Add self improvement pattern with two new prompt nodes

agent/README.md

@@ -1,28 +1,3 @@
 ## Refining the agent output
 
-Experiments modifying pipeline
+TODO: Table and document experiments
-
-| Model            | % Correct | % Change |
-|------------------|----------:|---------:|
-| BASELINE         | 33        | 0        |
-| Improv Prompt    | 39.96     | 0.21     |
-| Add Examples     | 44.67     | 0.35     |
-| Date             | 45.51     | 0.38     |
-| Chain of Thought | 43.38     | 0.31     |
-| Self-Critique    | 44.36     | 0.34     |
-
-Experiments with different model types:
-| Model                         | % Correct | % Change |
-|-------------------------------|----------:|---------:|
-| gpt-5-mini                    | 33        | 0        |
-| gpt-5.4-mini                  | 32.4      | -0.02    |
-| llama3.1:8b-instruct-q4_K_M   | ?         | ?        |
-| qwen3.5:9b                    | 0         | -100     |
-
-%age valid URLS
-| Model                         | Number    | % Age    |
-|-------------------------------|----------:|---------:|
-| gpt-5-mini                    | 22/405    | 5.43     |
-| gpt-5.4-mini                  | 29/278    | 10.43    |
-| llama3.1:8b-instruct-q4_K_M   | ?         | ?        |
-| qwen3.5:9b                    | 0         | 0        |

agent/agent.ts

@@ -11,13 +11,18 @@ import { loopEndConditional } from "./conditionals/loop_end";
 import { sort } from "./nodes/sort";
 import { triggerEventSetup } from "./nodes/triggerEventSetup";
 import { createEnsembleNode } from "./nodes/ensembleNode";
+import { selfEvalSetup } from "./nodes/selfEvalSetup";
 
 const triggerEventToolNode = createToolNode(triggerEventToolsByName);
+const peToolNode = createToolNode(triggerEventToolsByName);
 
 const normalisationModel = createModelNode([], "normalization.txt");
 const triggerEventModel = createModelNode(triggerEventToolsByName, "trigger.txt");
+const evaluationModel = createModelNode([], "eval.txt");
+const peModel = createModelNode(triggerEventToolsByName, "posteval.txt");
 
-const triggerEventToolConditional = createToolConditional("triggerEventToolNode", verificationSetup.name);
+const triggerEventToolConditional = createToolConditional("triggerEventToolNode", selfEvalSetup.name);
+const peToolConditional = createToolConditional("peToolNode", verificationSetup.name);
 
 const roNode = createEnsembleNode("ROBERTA", "roberta");
 const flNode = createEnsembleNode("FLAN", "flan");
@@ -33,6 +38,12 @@ const agent = new StateGraph(MessagesState)
   .addNode("triggerEventToolNode", triggerEventToolNode)
   .addNode("triggerEventModel", triggerEventModel)
 
+  .addNode(selfEvalSetup.name, selfEvalSetup)
+  .addNode("evaluationModel", evaluationModel)
+  
+  .addNode("peToolNode", peToolNode)
+  .addNode("peModel", peModel)
+
   .addNode(verificationSetup.name, verificationSetup)
 
   .addNode("roNode", roNode)
@@ -49,9 +60,16 @@ const agent = new StateGraph(MessagesState)
   .addEdge(triggerEventSetup.name, "triggerEventModel")
   
   // @ts-expect-error
-  .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", verificationSetup.name])
+  .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", selfEvalSetup.name])
   .addEdge("triggerEventToolNode", "triggerEventModel")
   
+  .addEdge(selfEvalSetup.name, "evaluationModel")
+  .addEdge("evaluationModel", "peModel")
+
+  // @ts-expect-error
+  .addConditionalEdges("peModel", peToolConditional, ["peToolNode", verificationSetup.name])
+  .addEdge("peToolNode", "peModel")
+  
   .addEdge(verificationSetup.name, "roNode")
   .addEdge(verificationSetup.name, "flNode")
   .addEdge(verificationSetup.name, "lrNode")

agent/nodes/model.ts

@@ -9,7 +9,7 @@ export function createModelNode(tools: any, promptPath: string): GraphNode<typeo
         const sysPrompt = await hydratePrompt(promptPath, state);
 
         const model = new ChatOpenAI({
-            model: "gpt-4.1-mini"
+            model: "gpt-5-mini"
         });
         const modelWithTools = model.bindTools(Object.values(tools));
 

agent/nodes/selfEvalSetup.ts

@@ -0,0 +1,21 @@
+import { GraphNode } from "@langchain/langgraph";
+import { MessagesState, ProposedTriggerEventArray } from "../state";
+import { logger } from "../utils/logger";
+import { queryScraper } from "../tools/webSearch";
+import { rankAndDisplayData } from "../tools/triggerEventTools";
+
+export const selfEvalSetup: GraphNode<typeof MessagesState> = async (state) => {
+  let genResponse = state.messages.at(-1)?.content.toString() ?? "";
+  const parsed = ProposedTriggerEventArray.parse(JSON.parse(genResponse));
+
+  for (let i = 0; i < parsed.length; i++) {
+    const search = parsed[i].SearchQuery
+    const data = await queryScraper(search);
+    const output = await rankAndDisplayData(data, search);
+
+    parsed[i].context = output;
+  }
+
+  return { evalTriggerEvent: parsed };
+
+};

agent/nodes/verificationSetup.ts

@@ -1,7 +1,8 @@
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState, ProposedTriggerEventArray } from "../state";
 import { logger } from "../utils/logger";
-import { jsonrepair } from 'jsonrepair'
+import { queryScraper } from "../tools/webSearch";
+import { rankAndDisplayData } from "../tools/triggerEventTools";
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
   //this is kinda doing two things, but having two nodes for it seems overkill
@@ -10,30 +11,7 @@ export const verificationSetup: GraphNode<typeof MessagesState> = async (state)
     logger.warn("No trigger events in memory, parsing")
 
     let genResponse = state.messages.at(-1)?.content.toString() ?? "";
-
+    const parsed = ProposedTriggerEventArray.parse(JSON.parse(genResponse));
-    const repaired = jsonrepair(genResponse);
-
-    let parsed;
-
-    try {
-      const json = JSON.parse(repaired);
-
-      if (Array.isArray(json)) {
-        parsed = ProposedTriggerEventArray.parse(json);
-      } else {
-        // try grab first value
-        const firstValue = Object.values(json)[0];
-
-        if (Array.isArray(firstValue)) {
-          parsed = ProposedTriggerEventArray.parse(firstValue);
-        } else {
-          throw new Error("No array found in JSON");
-        }
-      }
-    } catch (err: any) {
-      logger.error(`Failed to parse LLM response: ${err.message}`);
-      throw new Error(`Failed to parse LLM response: ${err}`);
-    }
 
     return { proposedTriggerEvent: parsed, proposedTriggerEventIndex: 0 };
   }

agent/package-lock.json

@@ -20,7 +20,6 @@
         "dotenv": "^17.2.3",
         "exponential-backoff": "^3.1.3",
         "fs": "^0.0.1-security",
-        "jsonrepair": "^3.13.3",
         "langchain": "^1.2.14",
         "selenium-webdriver": "^4.40.0",
         "tldts": "^7.0.23",
@@ -2076,15 +2075,6 @@
       "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
       "license": "ISC"
     },
-    "node_modules/jsonrepair": {
-      "version": "3.13.3",
-      "resolved": "https://registry.npmjs.org/jsonrepair/-/jsonrepair-3.13.3.tgz",
-      "integrity": "sha512-BTznj0owIt2CBAH/LTo7+1I5pMvl1e1033LRl/HUowlZmJOIhzC0zbX5bxMngLkfT4WnzPP26QnW5wMr2g9tsQ==",
-      "license": "ISC",
-      "bin": {
-        "jsonrepair": "bin/cli.js"
-      }
-    },
     "node_modules/jszip": {
       "version": "3.10.1",
       "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",

agent/package.json

@@ -24,7 +24,6 @@
     "dotenv": "^17.2.3",
     "exponential-backoff": "^3.1.3",
     "fs": "^0.0.1-security",
-    "jsonrepair": "^3.13.3",
     "langchain": "^1.2.14",
     "selenium-webdriver": "^4.40.0",
     "tldts": "^7.0.23",

agent/prompts/eval.txt

@@ -0,0 +1,30 @@
+You are an impartial and meticulous evaluator assessing LLM’s response based on key quality dimensions
+of honesty and usefulness. Your goal is to provide structured feedback that can be used to improve the
+response.
+Evaluation task: please follow these steps carefully:
+1. Analyze the response based on the three dimensions below.
+2. Provide justifications first: write a brief explanation justifying your assessment for each dimension.
+3. Assign scores after justification: assign a score from 1 (poor) to 10 (excellent) for each dimension
+based on your justification.
+4. Synthesize: provide a brief overall impression and the single most important suggestion for improve-
+ment, keeping in mind that explanation/honesty is the top priority, then followed by guidance.
+Critique dimensions (evaluate in this order):
+(1) Speficicity and usefullness: Can the proposed event be used to create a dataset of concrete events mapped to later
+disinformation.
+(2) Existance: Using the context provided, can the user be certain that the proposed trigger event actually happened
+(3) Causality: Is there a possible link from the proposed trigger event to the disinformaiton at hand
+Overall impression & key improvement suggestion: Briefly summarize the overall quality and state the
+most critical change needed to improve the response.
+
+Disinformation query: 
+###NTITLE###
+Disinformation date:
+###CDATE###
+
+LLM’s response to evaluate:
+###LM###
+
+Provided context:
+###VESEARCHES###
+
+Let's think it through step by step

agent/prompts/hydratePrompt.ts

@@ -15,6 +15,10 @@ export async function hydratePrompt(path: string, state: any) : Promise<string>
         raw = raw.replace("###LM###", state.messages.at(-1).content);
     }
 
+    if (raw.indexOf("###L2M###") != -1) {
+        raw = raw.replace("###L2M###", state.messages.at(-2).content);
+    }
+
     if (raw.indexOf("###NTITLE###") != -1) {
         raw = raw.replace("###NTITLE###", state.normalizedClaim);
     }
@@ -33,5 +37,12 @@ export async function hydratePrompt(path: string, state: any) : Promise<string>
         raw = raw.replace("###TESEARCH###", output)
     }
 
+    if (raw.indexOf("###VESEARCHES###") != -1) {
+        const output = state.evalTriggerEvent
+            .map(e => e.context)
+            .join("\n")
+        raw = raw.replace("###VESEARCHES###", output)
+    }
+
     return raw;
 }

agent/prompts/posteval.txt

@@ -0,0 +1,40 @@
+You are an expert editor tasked with making targeted improvements to an existing LLM’s response based
+on a specific critique with the primary goal of enhancing its score according to evaluation standards while
+preserving its strengths.
+Your revision task: generate a revised version of the existing response. Your goal is not to rewrite it
+completely, but to make precise edits only to address the specific weaknesses highlighted in the critique.
+Instructions for editing:
+- Identify specific flaws: carefully read the critique and pinpoint the exact issues raised (e.g., unclear
+explanation, vagueness, inappropriate responses, the key suggestion).
+- Perform minimal targeted edits: modify only the necessary sentences or paragraphs within the existing
+response to directly fix these identified flaws.
+- Strongly preserve strengths: crucially keep all other parts of the existing response intact. Do not
+rephrase, restructure, or remove sections that were not criticized or likely contributed positively to its
+initial score.
+- Ensure coherence: verify that your targeted edits integrate smoothly and do not introduce contradictions
+or awkward phrasing.
+Output requirements:
+- It should feel like a slightly polished or corrected version of the existing response, not a fundamentally
+different answer.
+- Do not mention the critique, scores, or the editing process. The output should be clean json that passes validation checks
+
+Again, use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,Date".
+Use tools available to you if further information is required
+
+Add no new events, only improve the existing items
+
+Disinformation query: 
+###NTITLE###
+Disinformation date:
+###CDATE###
+
+LLM’s response to improve:
+###L2M###
+
+Citique:
+###LM###
+
+This contains specific feedback, justifications, scores from 1 to 10, and potentially a key improvement
+suggestion. Focus on the justifications for low scores and the key suggestion.
+
+Let's think it through step by step

agent/prompts/trigger.txt

@@ -8,10 +8,6 @@ Produce up-to 5 specific "trigger events" that happened that could have led to t
 Remember the time frame of the disinformation campaign: ###CDATE###
 Include no information or events that would not have been available at the time. 
 
-You MEED TO use the tools available to you in order to produce up to date information on URL and search query, else you will be wrong and the analysis invalid.
-You NEED TO use the web search and open URL tools to ensure page validity or else all work upto this point will have to be discarded.
-
-
 Produce no more text other than the json.
 
 Include a concise but specific search query that can be looked up on a search engine in order to allow for the verification. 
@@ -30,9 +26,4 @@ Events will be reordered as part of processing, each statement must stand alone
 The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given.
 Analysis should only be completed for proposed events that would graner >0.7 points
 
-This pipeline is running well pasy your knowledge cutoff. 
-Any URLs will change signigicantly over time. 
-You MEED TO use the tools available to you in order to produce up to date information on URL and search query, else you will be wrong and the analysis invalid.
-You NEED TO use the web search and open URL tools to ensure page validity or else all work upto this point will have to be discarded.
-
 Lets go through it step by step

agent/state.ts

@@ -21,6 +21,7 @@ export const MessagesState = new StateSchema({
   date: z.string(),
   messages: MessagesValue,
   proposedTriggerEvent: ProposedTriggerEventArray,
+  evalTriggerEvent: ProposedTriggerEventArray,
   proposedTriggerEventIndex: z.int(),
   normalizedClaim: z.string(),
 });

run.sh

@@ -5,7 +5,7 @@ set -e
 run_agent () {
     echo "Starting LangGraph agent..."
     cd agent
-    npx @langchain/langgraph-cli dev --host 127.0.0.1
+    npx @langchain/langgraph-cli dev
 }
 
 run_ensemble_service () {

supporting/RAGAS_Service/requirements.txt

@@ -9,7 +9,6 @@ datasets
 # ROBERTA
 scikit-learn
 transformers[torch]
-sentence_transformers
 
 # Utils
 numpy

supporting/checker/checker.py

@@ -1,119 +0,0 @@
-import json
-import argparse
-from urllib.parse import urlparse
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.common.exceptions import WebDriverException, TimeoutException, StaleElementReferenceException
-from tqdm import tqdm
-
-def init_driver():
-    options = Options()
-    options.headless = True
-    options.add_argument("--disable-gpu")
-    options.add_argument("--no-sandbox")
-    options.add_argument("--headless")
-    options.add_argument("--disable-blink-features=AutomationControlled")
-    options.add_argument("--window-size=1920,1080")
-    prefs = {
-        "profile.managed_default_content_settings.images": 2,  # block images
-        "profile.default_content_setting_values.stylesheets": 2,  # block CSS
-        "profile.managed_default_content_settings.cookies": 2,  # optional
-    }
-    options.add_experimental_option("prefs", prefs)
-
-    driver = webdriver.Chrome(options=options)
-    driver.set_page_load_timeout(30)
-    return driver
-
-def is_root_url(url):
-    parsed = urlparse(url)
-    return parsed.path in ("", "/")
-
-def is_404_page(driver):
-    """Safely check for 404, handling stale elements."""
-    try:
-        title = driver.title.lower()
-        body_text = driver.find_element("tag name", "body").text.lower()
-        return "404" in title or "404" in body_text
-    except StaleElementReferenceException:
-        return False
-    except Exception:
-        return False
-
-def check_url_selenium(url):
-    driver = None
-    try:
-        driver = init_driver()
-        driver.get(url)
-        # 404 check
-        if is_404_page(driver):
-            return False, "404 page detected"
-        # Root URL after redirects
-        final_url = driver.current_url
-        if is_root_url(final_url):
-            return False, f"Redirected to root URL ({final_url})"
-        return True, None
-    except (WebDriverException, TimeoutException) as e:
-        return False, str(e)
-    finally:
-        if driver:
-            driver.quit()
-
-def process_event(event):
-    """Process an event only if score > 0.4."""
-    score = event.get("score", 0)
-    if score <= 0.4:
-        return None, False, "Score too low"
-    url = event.get("Url")
-    if not url:
-        return None, False, "No URL"
-    is_valid, error_msg = check_url_selenium(url)
-    event["url_valid"] = is_valid
-    return url, is_valid, error_msg
-
-def process_jsonl_file(file_path, max_workers=4):
-    invalid_urls = []
-    valid_urls = 0
-
-    # Gather events with score > 0.4
-    urls_to_check = []
-    with open(file_path, "r", encoding="utf-8") as f:
-        for line in f:
-            line_data = json.loads(line)
-            if line_data.get("status") != "success":
-                continue
-            for event in line_data.get("events", []):
-                if event.get("score", 0) > 0.4:
-                    urls_to_check.append(event)
-
-    total_urls = len(urls_to_check)
-
-    # ThreadPoolExecutor with tqdm progress bar
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        future_to_event = {executor.submit(process_event, e): e for e in urls_to_check}
-        for future in tqdm(as_completed(future_to_event), total=total_urls, desc="Checking URLs"):
-            url, is_valid, error_msg = future.result()
-            if not is_valid and url:
-                invalid_urls.append((url, error_msg))
-            else:
-                valid_urls += 1
-
-    # Summary
-    if invalid_urls:
-        print("\nList of invalid URLs and reasons:")
-        for url, err in invalid_urls:
-            print(f"{url} --> {err}")
-    print("\n=== URL Validation Summary ===")
-    print(f"Total URLs processed: {total_urls}")
-    print(f"Valid URLs (loaded successfully): {valid_urls}")
-    print(f"Invalid URLs: {len(invalid_urls)}")
-    
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Validate URLs in JSONL file events using Selenium")
-    parser.add_argument("file_path", type=str, help="Path to the JSONL file")
-    parser.add_argument("--workers", type=int, default=4, help="Number of parallel Selenium workers")
-    args = parser.parse_args()
-
-    process_jsonl_file(args.file_path, max_workers=args.workers)