From c89f73e138ed2061de9478b0d85d1ffab8fbcb22 Mon Sep 17 00:00:00 2001
From: William Jeynes <william.jeynes@hotmail.com>
Date: Thu, 12 Feb 2026 22:52:22 +0000
Subject: [PATCH] Implement RAGAS metrics

---
 agent/agent.ts                    |  9 ++++-----
 agent/nodes/dummyRagasMetrics.ts  | 11 -----------
 agent/nodes/normalizationSetup.ts |  2 --
 agent/nodes/ragasMetrics.ts       | 18 ++++++++++++++++++
 agent/nodes/verificationSetup.ts  |  3 +--
 agent/tools/webSearch.ts          |  2 +-
 6 files changed, 24 insertions(+), 21 deletions(-)
 delete mode 100644 agent/nodes/dummyRagasMetrics.ts
 create mode 100644 agent/nodes/ragasMetrics.ts

diff --git a/agent/agent.ts b/agent/agent.ts
index 6b0e4b7..c4be29d 100644
--- a/agent/agent.ts
+++ b/agent/agent.ts
@@ -4,9 +4,8 @@ import { createToolNode } from "./nodes/tool";
 import { createToolConditional } from "./conditionals/tool_end";
 import { normalizationSetup } from "./nodes/normalizationSetup";
 import { triggerEventToolsByName } from "./tools/triggerEventTools"
-import { createDummyModelNode } from "./nodes/dummyModel";
 import { verificationSetup } from "./nodes/verificationSetup";
-import { dummyRagasMetrics } from "./nodes/dummyRagasMetrics";
+import { ragasMetrics } from "./nodes/ragasMetrics";
 import { produceRanking } from "./nodes/produceRanking";
 import { createModelNode } from "./nodes/model";
 import { loopEndConditional } from "./conditionals/loop_end";
@@ -31,7 +30,7 @@ const agent = new StateGraph(MessagesState)
 
   .addNode(verificationSetup.name, verificationSetup)
   .addNode("verificationModel", verificationModel)
-  .addNode(dummyRagasMetrics.name, dummyRagasMetrics)
+  .addNode(ragasMetrics.name, ragasMetrics)
   .addNode(produceRanking.name, produceRanking)
   
   .addEdge(START, normalizationSetup.name)
@@ -43,9 +42,9 @@ const agent = new StateGraph(MessagesState)
   .addEdge("triggerEventToolNode", "triggerEventModel")
   
   .addEdge(verificationSetup.name, "verificationModel")
-  .addEdge(verificationSetup.name, dummyRagasMetrics.name)
+  .addEdge(verificationSetup.name, ragasMetrics.name)
 
-  .addEdge(dummyRagasMetrics.name, produceRanking.name)
+  .addEdge(ragasMetrics.name, produceRanking.name)
   .addEdge("verificationModel", produceRanking.name)
 
   // @ts-expect-error
diff --git a/agent/nodes/dummyRagasMetrics.ts b/agent/nodes/dummyRagasMetrics.ts
deleted file mode 100644
index 33ee364..0000000
--- a/agent/nodes/dummyRagasMetrics.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-import { GraphNode } from "@langchain/langgraph";
-import { MessagesState } from "../state";
-import { AIMessage, HumanMessage } from "@langchain/core/messages";
-
-export const dummyRagasMetrics: GraphNode<typeof MessagesState> = async (state) => {
-  //TODO: get ragas metrics
-  
-  return {
-    messages: [ new AIMessage("RAGASSED : " + state.messages.at(-1)?.content)]
-  };
-};
\ No newline at end of file
diff --git a/agent/nodes/normalizationSetup.ts b/agent/nodes/normalizationSetup.ts
index 02a03a1..bc2d661 100644
--- a/agent/nodes/normalizationSetup.ts
+++ b/agent/nodes/normalizationSetup.ts
@@ -6,8 +6,6 @@ import { rankFromCSV } from "../tools/clan/retreiveExamples";
 export const normalizationSetup: GraphNode<typeof MessagesState> = async (state) => {
   let similarityResults = await rankFromCSV(state.disinformationTitle)
 
-  console.log(similarityResults)
-
   let messages : BaseMessage[] = similarityResults.map((item) => {
     return new AIMessage(`Original Claim: ${item.rawtext}. \n\n Normalised Claim: ${item.cleantext}`)
   })
diff --git a/agent/nodes/ragasMetrics.ts b/agent/nodes/ragasMetrics.ts
new file mode 100644
index 0000000..ee72693
--- /dev/null
+++ b/agent/nodes/ragasMetrics.ts
@@ -0,0 +1,18 @@
+import { GraphNode } from "@langchain/langgraph";
+import { MessagesState } from "../state";
+import { AIMessage, HumanMessage } from "@langchain/core/messages";
+import { evaluateWithRagas } from "../tools/ragasCall";
+
+export const ragasMetrics: GraphNode<typeof MessagesState> = async (state) => {
+  const question = "A possible trigger event for: " + state.disinformationTitle //Should it be raw, or normalized?
+  const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
+  const contexts = state.proposedTriggerEvent[state.proposedTriggerEventIndex].context?.split("^^^") ?? []
+  
+  console.log(contexts)
+  
+  const results = await evaluateWithRagas({question, answer, contexts})
+  
+  return {
+    messages: [ new AIMessage("RAGAS:" + results.faithfulness)]
+  };
+};
\ No newline at end of file
diff --git a/agent/nodes/verificationSetup.ts b/agent/nodes/verificationSetup.ts
index 364365c..700b1d3 100644
--- a/agent/nodes/verificationSetup.ts
+++ b/agent/nodes/verificationSetup.ts
@@ -6,8 +6,7 @@ import { rankAndDisplayData } from "../tools/triggerEventTools";
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
   //this is kinda doing two things, but having two nodes for it seems overkill
-  console.log(state.proposedTriggerEvent)
-  console.log(state.proposedTriggerEventIndex)
+
   if (state.proposedTriggerEvent == undefined) {
     logger.warn("No trigger events in memory, parsing")
 
diff --git a/agent/tools/webSearch.ts b/agent/tools/webSearch.ts
index 33bc508..833798e 100644
--- a/agent/tools/webSearch.ts
+++ b/agent/tools/webSearch.ts
@@ -48,7 +48,7 @@ export async function queryScraper(query: string): Promise<string[]> {
         const desc = (item.description ?? "").trim();
         const link = (item.url ?? "").trim();
 
-        return `- ${title}\n  ${desc}\n  ${link}`;
+        return `^^^ ${title}\n  ${desc}\n  ${link}`;
     });
 
     return lines;