Improve response extraction

agent/.env.example

@@ -3,4 +3,5 @@ LANGSMITH_TRACING=true
 LANGSMITH_API_KEY=123456
 LANGSMITH_ENDPOINT=https://eu.api.smith.langchain.com
 SCRAPER_INSTANCE=https://example.com
-SCRAPER_PARAM_ANYTHING=else
+SCRAPER_PARAM_ANYTHING=else
+RANKING_URL=http://localhost:8000/evaluate

agent/nodes/model.ts

@@ -1,25 +1,28 @@
-import { HumanMessage, SystemMessage } from "@langchain/core/messages";
+import { SystemMessage } from "@langchain/core/messages";
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState } from "../state";
-import { ChatOpenAI } from "@langchain/openai"
+import { ChatOllama } from "@langchain/ollama";
 import { hydratePrompt } from "../prompts/hydratePrompt";
+import { logger } from "../utils/logger";
 
 export function createModelNode(tools: any, promptPath: string): GraphNode<typeof MessagesState> {
     return async (state) => {
         const sysPrompt = await hydratePrompt(promptPath, state);
 
-        const model = new ChatOpenAI({
-            model: "gpt-5.4-mini"
+        const model = new ChatOllama({
+            model: "llama3.1:8b-instruct-q4_K_M",
+            temperature: 0.3
         });
+
         const modelWithTools = model.bindTools(Object.values(tools));
 
         const response = await modelWithTools.invoke([
-            new SystemMessage(
-                sysPrompt
-            ),
+            new SystemMessage(sysPrompt),
             ...state.messages,
         ]);
 
+        logger.error(response);
+
         return {
             messages: [response]
         };

agent/nodes/triggerEventSetup.ts

@@ -3,9 +3,17 @@ import { MessagesState } from "../state";
 import { AIMessage, BaseMessage } from "@langchain/core/messages";
 import { rankExampleTriggerEvents } from "../tools/retreiveExamples";
 
+function extractTE(text: string) {
+  const match = text.match(/<norm>([\s\S]*?)<\/norm>/);
+  if (!match) throw new Error("Nothing found between <norm> tags");
+  return match[1].trim();
+}
+
+
 export const triggerEventSetup: GraphNode<typeof MessagesState> = async (state) => {
-  let nc = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
-  
+  let raw = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
+  let nc = extractTE(raw.toString())
+
   //Now give in-context examples. hopwfully we can self-teach?
   let similarityResults = await rankExampleTriggerEvents(state.disinformationTitle)
 

agent/nodes/verificationSetup.ts

@@ -1,34 +1,60 @@
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState, ProposedTriggerEventArray } from "../state";
 import { logger } from "../utils/logger";
-import { jsonrepair } from 'jsonrepair'
+import { jsonrepair } from 'jsonrepair';
+
+function extractJSON(text: string) {
+  const match = text.match(/<json>([\s\S]*?)<\/json>/);
+  if (!match) throw new Error("No JSON found between <json> tags");
+  return match[1].trim();
+}
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
-  //this is kinda doing two things, but having two nodes for it seems overkill
-
   if (state.proposedTriggerEvent == undefined) {
-    logger.warn("No trigger events in memory, parsing")
+    logger.warn("No trigger events in memory, parsing");
 
-    let genResponse = state.messages.at(-1)?.content.toString() ?? "";
+    const genResponse = state.messages.at(-1)?.content.toString() ?? "";
 
-    const repaired = jsonrepair(genResponse);
-
-    const parsed = ProposedTriggerEventArray.parse(JSON.parse(repaired));
-
-    for (let i = 0; i < parsed.length; i++) {
-      const search = parsed[i].SearchQuery
-      // const data = await queryScraper(search);
-      // const output = await rankAndDisplayData(data, search);
-
-      // parsed[i].context = output;
-      parsed[i].context = "NONE"
+    let repaired: string;
+    try {
+      let extracted = extractJSON(genResponse)
+      repaired = jsonrepair(extracted);
+    } catch (repairErr: any) {
+      logger.error("Failed to repair JSON from LLM response.");
+      logger.error("Original LLM response:\n%s", genResponse);
+      throw new Error(`JSON repair failed: ${repairErr.message}`);
     }
-    
+
+    let parsed;
+    try {
+      const json = JSON.parse(repaired);
+
+      if (Array.isArray(json)) {
+        parsed = ProposedTriggerEventArray.parse(json);
+      } else {
+        // try grab first value
+        const firstValue = Object.values(json)[0];
+
+        if (Array.isArray(firstValue)) {
+          parsed = ProposedTriggerEventArray.parse(firstValue);
+        } else {
+          logger.error("No array found in JSON after parsing.");
+          logger.error("Repaired JSON:\n%s", repaired);
+          logger.error("Original LLM response:\n%s", genResponse);
+          throw new Error("No array found in JSON structure");
+        }
+      }
+    } catch (parseErr: any) {
+      logger.error("Failed to parse LLM response to JSON or validate array.");
+      logger.error("Repaired JSON:\n%s", repaired);
+      logger.error("Original LLM response:\n%s", genResponse);
+      throw new Error(`Parsing failed: ${parseErr.message}`);
+    }
+
     return { proposedTriggerEvent: parsed, proposedTriggerEventIndex: 0 };
-  }
-  else {
-    logger.info("Trigger event index %s", state.proposedTriggerEventIndex+1)
-    
-    return { proposedTriggerEvent: state.proposedTriggerEvent, proposedTriggerEventIndex: state.proposedTriggerEventIndex+1 };
+  } else {
+    logger.info("Trigger event index %s", state.proposedTriggerEventIndex + 1);
+
+    return { proposedTriggerEvent: state.proposedTriggerEvent, proposedTriggerEventIndex: state.proposedTriggerEventIndex + 1 };
   }
 };

agent/package.json

@@ -17,6 +17,7 @@
     "@langchain/core": "^1.1.17",
     "@langchain/langgraph": "^1.1.2",
     "@langchain/langgraph-sdk": "^1.5.5",
+    "@langchain/ollama": "^1.2.6",
     "@langchain/openai": "^1.2.3",
     "axios": "^1.13.5",
     "compute-cosine-similarity": "^1.1.0",

agent/prompts/normalization.txt

@@ -16,4 +16,7 @@ Relevent examples are included in preceeding messages, use these as exact inspir
 The claim to normalize is:
 ###TITLE###
 
-Produce no other text other than the condensed claim.
+Produce no other text other than the condensed claim, surrounded <norm></norm>
+
+For example: BREAKING: the sky is green!
+Becomes: <norm>The sky is green</norm>

agent/prompts/relation.txt

@@ -1,9 +0,0 @@
-Could the following real-world event:
-###TECLAIM###
-
-Be a trigger for the following disinformation:
-###TITLE###
-
-Respond with "RELATION", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
-
-Ignore wether the event happened or not, purely consider the likiness of causation

agent/prompts/trigger.txt

@@ -17,6 +17,15 @@ Include a url to a source for your trigger event (not a web search, a specific u
 Include the date that the event happened ("March 2022" for exmaple)
 
 Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,Date".
+Return ONLY JSON, no extra text. Wrap it like this:
+<json>
+[
+    {
+        "Event": "Example"
+        ...
+    }
+]
+</json>
 
 Multiple tool invocations should be requested at once, if applicable.
 Use your abilities to look between the lines and produce some insightful analysis, thinking both short and long term.
@@ -26,8 +35,8 @@ Events will be reordered as part of processing, each statement must stand alone
 The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given.
 Analysis should only be completed for proposed events that would graner >0.7 points
 
-First, consider a range of directions in which the proposed disinformation could have been influenced by.
-Then, research these directions in turn, using the tools at hand.
-Finally, refine your proposed "trigger event" until it is specific, quantifiable and backed up by evidence.
+Since URLs change frequently, use tools to retreive up to date informaiton everytime, provided examples or existing knowledge will be wrong or out of date.
+
+Remember to return just json enclosed by <json></json>
 
 Lets go through it step by step

agent/prompts/verify.txt

@@ -1,8 +0,0 @@
-Do the search results cited below
-###TESEARCH###
-Support the idea that the following happened:
-###TECLAIM###
-
-Respond with "CONFIDENCE", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
-
-Dates can be off by a few days, that would still be valid

agent/tools/ensembleCall.ts

@@ -7,7 +7,7 @@ export async function evaluateWithEnsemble({
   answer: string;
   method: string
 }): Promise<{ validProb: number; invalidProb: number; }> {
-  const res = await axios.post("http://localhost:8000/evaluate", {
+    const res = await axios.post(process.env.RANKING_URL ?? "http://localhost:8000/evaluate", {
     answer,
     method
   }, {timeout: 0});
@@ -18,11 +18,15 @@ export async function evaluateWithEnsemble({
   return {validProb, invalidProb};
 }
 
-// let res = await evaluateWithRoberta({answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
+// import dotenv from "dotenv";
+
+// dotenv.config();
+
+// let res = await evaluateWithEnsemble({method:"flan" ,answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
 // console.log(res)
 
-// res = await evaluateWithRoberta({answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
+// res = await evaluateWithEnsemble({method:"roberta" ,answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
 // console.log(res)
 
-// res = await evaluateWithRoberta({answer: "The COVID-19 Pandemic"});
+// res = await evaluateWithEnsemble({method:"logreg" ,answer: "The COVID-19 Pandemic"});
 // console.log(res)

agent/tools/ragasCall.ts

@@ -1,22 +0,0 @@
-import axios from "axios";
-
-export async function evaluateWithRagas({
-  question,
-  answer,
-  contexts,
-}: {
-  question: string;
-  answer: string;
-  contexts: string[];
-}) {
-  const res = await axios.post("http://localhost:8001/evaluate", {
-    question,
-    answer,
-    contexts,
-  });
-
-  return res.data;
-}
-
-// let res = await evaluateWithRagas({question: "Who was Bill Nye", answer: "Bill Nye was a Scientist", contexts: ["Bill nye was a Scientist"]});
-// console.log(res)

agent/tools/webpageFetch.ts

@@ -8,7 +8,7 @@ export async function extractWebpageContent(url: string): Promise<string[]> {
     const response = await backOff(async () => {
       return await extractWebpageContentWorker(url);
     }, {
-      numOfAttempts: 5,
+      numOfAttempts: 10,
       startingDelay: 500,
       timeMultiple: 2,
       jitter: "full",
@@ -26,6 +26,9 @@ async function extractWebpageContentWorker(url: string): Promise<string[]> {
   try {
     const options = new firefox.Options();
     options.addArguments("--headless");
+    options.addArguments("--disable-gpu");
+    options.addArguments("--no-sandbox"); // Linux sandbox issues
+    options.addArguments("--disable-dev-shm-usage"); // /dev/shm issues
     driver = await new Builder()
       .forBrowser(Browser.FIREFOX)
       .setFirefoxOptions(options)

supporting/RAGAS_Service/ensemble_service.py

@@ -92,7 +92,7 @@ LABEL_TO_INT = {v: k for k, v in INT_TO_LABEL.items()}
 flan_tokenizer = AutoTokenizer.from_pretrained(FLAN_PATH)
 flan_model = AutoModelForSeq2SeqLM.from_pretrained(FLAN_PATH)
 
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = torch.device("cpu")
 flan_model.to(device)
 flan_model.eval()
 

supporting/RAGAS_Service/requirements.txt

@@ -9,6 +9,7 @@ datasets
 # ROBERTA
 scikit-learn
 transformers[torch]
+sentence_transformers
 
 # Utils
 numpy

supporting/Wrapper/run.ts

@@ -17,7 +17,7 @@ const AGENT_NAME = process.env.AGENT ?? "agent";
  */
 const MODE = process.env.MODE ?? "claim";
 
-const MAX_CONCURRENCY = 5;
+const MAX_CONCURRENCY = 1;
 
 const client = new Client({ apiUrl: API_URL });