Add self improvement pattern with two new prompt nodes

agent/.env.example

@@ -4,4 +4,3 @@ LANGSMITH_API_KEY=123456
 LANGSMITH_ENDPOINT=https://eu.api.smith.langchain.com
 SCRAPER_INSTANCE=https://example.com
 SCRAPER_PARAM_ANYTHING=else
-RANKING_URL=http://localhost:8000/evaluate

agent/agent.ts

@@ -11,13 +11,18 @@ import { loopEndConditional } from "./conditionals/loop_end";
 import { sort } from "./nodes/sort";
 import { triggerEventSetup } from "./nodes/triggerEventSetup";
 import { createEnsembleNode } from "./nodes/ensembleNode";
+import { selfEvalSetup } from "./nodes/selfEvalSetup";
 
 const triggerEventToolNode = createToolNode(triggerEventToolsByName);
+const peToolNode = createToolNode(triggerEventToolsByName);
 
 const normalisationModel = createModelNode([], "normalization.txt");
 const triggerEventModel = createModelNode(triggerEventToolsByName, "trigger.txt");
+const evaluationModel = createModelNode([], "eval.txt");
+const peModel = createModelNode(triggerEventToolsByName, "posteval.txt");
 
-const triggerEventToolConditional = createToolConditional("triggerEventToolNode", verificationSetup.name);
+const triggerEventToolConditional = createToolConditional("triggerEventToolNode", selfEvalSetup.name);
+const peToolConditional = createToolConditional("peToolNode", verificationSetup.name);
 
 const roNode = createEnsembleNode("ROBERTA", "roberta");
 const flNode = createEnsembleNode("FLAN", "flan");
@@ -33,6 +38,12 @@ const agent = new StateGraph(MessagesState)
   .addNode("triggerEventToolNode", triggerEventToolNode)
   .addNode("triggerEventModel", triggerEventModel)
 
+  .addNode(selfEvalSetup.name, selfEvalSetup)
+  .addNode("evaluationModel", evaluationModel)
+  
+  .addNode("peToolNode", peToolNode)
+  .addNode("peModel", peModel)
+
   .addNode(verificationSetup.name, verificationSetup)
 
   .addNode("roNode", roNode)
@@ -49,9 +60,16 @@ const agent = new StateGraph(MessagesState)
   .addEdge(triggerEventSetup.name, "triggerEventModel")
   
   // @ts-expect-error
-  .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", verificationSetup.name])
+  .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", selfEvalSetup.name])
   .addEdge("triggerEventToolNode", "triggerEventModel")
   
+  .addEdge(selfEvalSetup.name, "evaluationModel")
+  .addEdge("evaluationModel", "peModel")
+
+  // @ts-expect-error
+  .addConditionalEdges("peModel", peToolConditional, ["peToolNode", verificationSetup.name])
+  .addEdge("peToolNode", "peModel")
+  
   .addEdge(verificationSetup.name, "roNode")
   .addEdge(verificationSetup.name, "flNode")
   .addEdge(verificationSetup.name, "lrNode")

agent/nodes/model.ts

@@ -1,28 +1,25 @@
-import { SystemMessage } from "@langchain/core/messages";
+import { HumanMessage, SystemMessage } from "@langchain/core/messages";
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState } from "../state";
-import { ChatOllama } from "@langchain/ollama";
+import { ChatOpenAI } from "@langchain/openai"
 import { hydratePrompt } from "../prompts/hydratePrompt";
-import { logger } from "../utils/logger";
 
 export function createModelNode(tools: any, promptPath: string): GraphNode<typeof MessagesState> {
     return async (state) => {
         const sysPrompt = await hydratePrompt(promptPath, state);
 
-        const model = new ChatOllama({
+        const model = new ChatOpenAI({
-            model: "llama3.1:8b-instruct-q4_K_M",
+            model: "gpt-5-mini"
-            temperature: 0.3
         });
-
         const modelWithTools = model.bindTools(Object.values(tools));
 
         const response = await modelWithTools.invoke([
-            new SystemMessage(sysPrompt),
+            new SystemMessage(
+                sysPrompt
+            ),
             ...state.messages,
         ]);
 
-        logger.error(response);
-
         return {
             messages: [response]
         };

agent/nodes/selfEvalSetup.ts

@@ -0,0 +1,21 @@
+import { GraphNode } from "@langchain/langgraph";
+import { MessagesState, ProposedTriggerEventArray } from "../state";
+import { logger } from "../utils/logger";
+import { queryScraper } from "../tools/webSearch";
+import { rankAndDisplayData } from "../tools/triggerEventTools";
+
+export const selfEvalSetup: GraphNode<typeof MessagesState> = async (state) => {
+  let genResponse = state.messages.at(-1)?.content.toString() ?? "";
+  const parsed = ProposedTriggerEventArray.parse(JSON.parse(genResponse));
+
+  for (let i = 0; i < parsed.length; i++) {
+    const search = parsed[i].SearchQuery
+    const data = await queryScraper(search);
+    const output = await rankAndDisplayData(data, search);
+
+    parsed[i].context = output;
+  }
+
+  return { evalTriggerEvent: parsed };
+
+};

agent/nodes/triggerEventSetup.ts

@@ -3,16 +3,8 @@ import { MessagesState } from "../state";
 import { AIMessage, BaseMessage } from "@langchain/core/messages";
 import { rankExampleTriggerEvents } from "../tools/retreiveExamples";
 
-function extractTE(text: string) {
-  const match = text.match(/<norm>([\s\S]*?)<\/norm>/);
-  if (!match) throw new Error("Nothing found between <norm> tags");
-  return match[1].trim();
-}
-
-
 export const triggerEventSetup: GraphNode<typeof MessagesState> = async (state) => {
-  let raw = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
+  let nc = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
-  let nc = extractTE(raw.toString())
   
   //Now give in-context examples. hopwfully we can self-teach?
   let similarityResults = await rankExampleTriggerEvents(state.disinformationTitle)

agent/nodes/verificationSetup.ts

@@ -1,59 +1,22 @@
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState, ProposedTriggerEventArray } from "../state";
 import { logger } from "../utils/logger";
-import { jsonrepair } from 'jsonrepair';
+import { queryScraper } from "../tools/webSearch";
-
+import { rankAndDisplayData } from "../tools/triggerEventTools";
-function extractJSON(text: string) {
-  const match = text.match(/<json>([\s\S]*?)<\/json>/);
-  if (!match) throw new Error("No JSON found between <json> tags");
-  return match[1].trim();
-}
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
+  //this is kinda doing two things, but having two nodes for it seems overkill
+
   if (state.proposedTriggerEvent == undefined) {
-    logger.warn("No trigger events in memory, parsing");
+    logger.warn("No trigger events in memory, parsing")
 
-    const genResponse = state.messages.at(-1)?.content.toString() ?? "";
+    let genResponse = state.messages.at(-1)?.content.toString() ?? "";
-
+    const parsed = ProposedTriggerEventArray.parse(JSON.parse(genResponse));
-    let repaired: string;
-    try {
-      let extracted = extractJSON(genResponse)
-      repaired = jsonrepair(extracted);
-    } catch (repairErr: any) {
-      logger.error("Failed to repair JSON from LLM response.");
-      logger.error("Original LLM response:\n%s", genResponse);
-      throw new Error(`JSON repair failed: ${repairErr.message}`);
-    }
-
-    let parsed;
-    try {
-      const json = JSON.parse(repaired);
-
-      if (Array.isArray(json)) {
-        parsed = ProposedTriggerEventArray.parse(json);
-      } else {
-        // try grab first value
-        const firstValue = Object.values(json)[0];
-
-        if (Array.isArray(firstValue)) {
-          parsed = ProposedTriggerEventArray.parse(firstValue);
-        } else {
-          logger.error("No array found in JSON after parsing.");
-          logger.error("Repaired JSON:\n%s", repaired);
-          logger.error("Original LLM response:\n%s", genResponse);
-          throw new Error("No array found in JSON structure");
-        }
-      }
-    } catch (parseErr: any) {
-      logger.error("Failed to parse LLM response to JSON or validate array.");
-      logger.error("Repaired JSON:\n%s", repaired);
-      logger.error("Original LLM response:\n%s", genResponse);
-      throw new Error(`Parsing failed: ${parseErr.message}`);
-    }
 
     return { proposedTriggerEvent: parsed, proposedTriggerEventIndex: 0 };
-  } else {
+  }
-    logger.info("Trigger event index %s", state.proposedTriggerEventIndex + 1);
+  else {
+    logger.info("Trigger event index %s", state.proposedTriggerEventIndex+1)
     
     return { proposedTriggerEvent: state.proposedTriggerEvent, proposedTriggerEventIndex: state.proposedTriggerEventIndex+1 };
   }

agent/package.json

@@ -17,7 +17,6 @@
     "@langchain/core": "^1.1.17",
     "@langchain/langgraph": "^1.1.2",
     "@langchain/langgraph-sdk": "^1.5.5",
-    "@langchain/ollama": "^1.2.6",
     "@langchain/openai": "^1.2.3",
     "axios": "^1.13.5",
     "compute-cosine-similarity": "^1.1.0",
@@ -25,7 +24,6 @@
     "dotenv": "^17.2.3",
     "exponential-backoff": "^3.1.3",
     "fs": "^0.0.1-security",
-    "jsonrepair": "^3.13.3",
     "langchain": "^1.2.14",
     "selenium-webdriver": "^4.40.0",
     "tldts": "^7.0.23",

agent/prompts/eval.txt

@@ -0,0 +1,30 @@
+You are an impartial and meticulous evaluator assessing LLM’s response based on key quality dimensions
+of honesty and usefulness. Your goal is to provide structured feedback that can be used to improve the
+response.
+Evaluation task: please follow these steps carefully:
+1. Analyze the response based on the three dimensions below.
+2. Provide justifications first: write a brief explanation justifying your assessment for each dimension.
+3. Assign scores after justification: assign a score from 1 (poor) to 10 (excellent) for each dimension
+based on your justification.
+4. Synthesize: provide a brief overall impression and the single most important suggestion for improve-
+ment, keeping in mind that explanation/honesty is the top priority, then followed by guidance.
+Critique dimensions (evaluate in this order):
+(1) Speficicity and usefullness: Can the proposed event be used to create a dataset of concrete events mapped to later
+disinformation.
+(2) Existance: Using the context provided, can the user be certain that the proposed trigger event actually happened
+(3) Causality: Is there a possible link from the proposed trigger event to the disinformaiton at hand
+Overall impression & key improvement suggestion: Briefly summarize the overall quality and state the
+most critical change needed to improve the response.
+
+Disinformation query: 
+###NTITLE###
+Disinformation date:
+###CDATE###
+
+LLM’s response to evaluate:
+###LM###
+
+Provided context:
+###VESEARCHES###
+
+Let's think it through step by step

agent/prompts/hydratePrompt.ts

@@ -15,6 +15,10 @@ export async function hydratePrompt(path: string, state: any) : Promise<string>
         raw = raw.replace("###LM###", state.messages.at(-1).content);
     }
 
+    if (raw.indexOf("###L2M###") != -1) {
+        raw = raw.replace("###L2M###", state.messages.at(-2).content);
+    }
+
     if (raw.indexOf("###NTITLE###") != -1) {
         raw = raw.replace("###NTITLE###", state.normalizedClaim);
     }
@@ -33,5 +37,12 @@ export async function hydratePrompt(path: string, state: any) : Promise<string>
         raw = raw.replace("###TESEARCH###", output)
     }
 
+    if (raw.indexOf("###VESEARCHES###") != -1) {
+        const output = state.evalTriggerEvent
+            .map(e => e.context)
+            .join("\n")
+        raw = raw.replace("###VESEARCHES###", output)
+    }
+
     return raw;
 }

agent/prompts/normalization.txt

@@ -16,7 +16,4 @@ Relevent examples are included in preceeding messages, use these as exact inspir
 The claim to normalize is:
 ###TITLE###
 
-Produce no other text other than the condensed claim, surrounded <norm></norm>
+Produce no other text other than the condensed claim.
-
-For example: BREAKING: the sky is green!
-Becomes: <norm>The sky is green</norm>

agent/prompts/posteval.txt

@@ -0,0 +1,40 @@
+You are an expert editor tasked with making targeted improvements to an existing LLM’s response based
+on a specific critique with the primary goal of enhancing its score according to evaluation standards while
+preserving its strengths.
+Your revision task: generate a revised version of the existing response. Your goal is not to rewrite it
+completely, but to make precise edits only to address the specific weaknesses highlighted in the critique.
+Instructions for editing:
+- Identify specific flaws: carefully read the critique and pinpoint the exact issues raised (e.g., unclear
+explanation, vagueness, inappropriate responses, the key suggestion).
+- Perform minimal targeted edits: modify only the necessary sentences or paragraphs within the existing
+response to directly fix these identified flaws.
+- Strongly preserve strengths: crucially keep all other parts of the existing response intact. Do not
+rephrase, restructure, or remove sections that were not criticized or likely contributed positively to its
+initial score.
+- Ensure coherence: verify that your targeted edits integrate smoothly and do not introduce contradictions
+or awkward phrasing.
+Output requirements:
+- It should feel like a slightly polished or corrected version of the existing response, not a fundamentally
+different answer.
+- Do not mention the critique, scores, or the editing process. The output should be clean json that passes validation checks
+
+Again, use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,Date".
+Use tools available to you if further information is required
+
+Add no new events, only improve the existing items
+
+Disinformation query: 
+###NTITLE###
+Disinformation date:
+###CDATE###
+
+LLM’s response to improve:
+###L2M###
+
+Citique:
+###LM###
+
+This contains specific feedback, justifications, scores from 1 to 10, and potentially a key improvement
+suggestion. Focus on the justifications for low scores and the key suggestion.
+
+Let's think it through step by step

agent/prompts/relation.txt

@@ -0,0 +1,9 @@
+Could the following real-world event:
+###TECLAIM###
+
+Be a trigger for the following disinformation:
+###TITLE###
+
+Respond with "RELATION", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
+
+Ignore wether the event happened or not, purely consider the likiness of causation

agent/prompts/trigger.txt

@@ -17,15 +17,6 @@ Include a url to a source for your trigger event (not a web search, a specific u
 Include the date that the event happened ("March 2022" for exmaple)
 
 Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,Date".
-Return ONLY JSON, no extra text. Wrap it like this:
-<json>
-[
-    {
-        "Event": "Example"
-        ...
-    }
-]
-</json>
 
 Multiple tool invocations should be requested at once, if applicable.
 Use your abilities to look between the lines and produce some insightful analysis, thinking both short and long term.
@@ -35,8 +26,4 @@ Events will be reordered as part of processing, each statement must stand alone
 The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given.
 Analysis should only be completed for proposed events that would graner >0.7 points
 
-Since URLs change frequently, use tools to retreive up to date informaiton everytime, provided examples or existing knowledge will be wrong or out of date.
-
-Remember to return just json enclosed by <json></json>
-
 Lets go through it step by step

agent/prompts/verify.txt

@@ -0,0 +1,8 @@
+Do the search results cited below
+###TESEARCH###
+Support the idea that the following happened:
+###TECLAIM###
+
+Respond with "CONFIDENCE", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
+
+Dates can be off by a few days, that would still be valid

agent/state.ts

@@ -21,6 +21,7 @@ export const MessagesState = new StateSchema({
   date: z.string(),
   messages: MessagesValue,
   proposedTriggerEvent: ProposedTriggerEventArray,
+  evalTriggerEvent: ProposedTriggerEventArray,
   proposedTriggerEventIndex: z.int(),
   normalizedClaim: z.string(),
 });

agent/tools/ensembleCall.ts

@@ -7,7 +7,7 @@ export async function evaluateWithEnsemble({
   answer: string;
   method: string
 }): Promise<{ validProb: number; invalidProb: number; }> {
-    const res = await axios.post(process.env.RANKING_URL ?? "http://localhost:8000/evaluate", {
+  const res = await axios.post("http://localhost:8000/evaluate", {
     answer,
     method
   }, {timeout: 0});
@@ -18,15 +18,11 @@ export async function evaluateWithEnsemble({
   return {validProb, invalidProb};
 }
 
-// import dotenv from "dotenv";
+// let res = await evaluateWithRoberta({answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
-
-// dotenv.config();
-
-// let res = await evaluateWithEnsemble({method:"flan" ,answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
 // console.log(res)
 
-// res = await evaluateWithEnsemble({method:"roberta" ,answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
+// res = await evaluateWithRoberta({answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
 // console.log(res)
 
-// res = await evaluateWithEnsemble({method:"logreg" ,answer: "The COVID-19 Pandemic"});
+// res = await evaluateWithRoberta({answer: "The COVID-19 Pandemic"});
 // console.log(res)

agent/tools/ragasCall.ts

@@ -0,0 +1,22 @@
+import axios from "axios";
+
+export async function evaluateWithRagas({
+  question,
+  answer,
+  contexts,
+}: {
+  question: string;
+  answer: string;
+  contexts: string[];
+}) {
+  const res = await axios.post("http://localhost:8001/evaluate", {
+    question,
+    answer,
+    contexts,
+  });
+
+  return res.data;
+}
+
+// let res = await evaluateWithRagas({question: "Who was Bill Nye", answer: "Bill Nye was a Scientist", contexts: ["Bill nye was a Scientist"]});
+// console.log(res)

agent/tools/webpageFetch.ts

@@ -26,9 +26,6 @@ async function extractWebpageContentWorker(url: string): Promise<string[]> {
   try {
     const options = new firefox.Options();
     options.addArguments("--headless");
-    options.addArguments("--disable-gpu");
-    options.addArguments("--no-sandbox"); // Linux sandbox issues
-    options.addArguments("--disable-dev-shm-usage"); // /dev/shm issues
     driver = await new Builder()
       .forBrowser(Browser.FIREFOX)
       .setFirefoxOptions(options)

supporting/RAGAS_Service/ensemble_service.py

@@ -92,7 +92,7 @@ LABEL_TO_INT = {v: k for k, v in INT_TO_LABEL.items()}
 flan_tokenizer = AutoTokenizer.from_pretrained(FLAN_PATH)
 flan_model = AutoModelForSeq2SeqLM.from_pretrained(FLAN_PATH)
 
-device = torch.device("cpu")
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 flan_model.to(device)
 flan_model.eval()
 

supporting/RAGAS_Service/requirements.txt

@@ -9,7 +9,6 @@ datasets
 # ROBERTA
 scikit-learn
 transformers[torch]
-sentence_transformers
 
 # Utils
 numpy

supporting/Wrapper/run.ts

@@ -17,7 +17,7 @@ const AGENT_NAME = process.env.AGENT ?? "agent";
  */
 const MODE = process.env.MODE ?? "claim";
 
-const MAX_CONCURRENCY = 1;
+const MAX_CONCURRENCY = 5;
 
 const client = new Client({ apiUrl: API_URL });