Implement RAGAS metrics

agent/agent.ts

@@ -4,9 +4,8 @@ import { createToolNode } from "./nodes/tool";
 import { createToolConditional } from "./conditionals/tool_end";
 import { normalizationSetup } from "./nodes/normalizationSetup";
 import { triggerEventToolsByName } from "./tools/triggerEventTools"
-import { createDummyModelNode } from "./nodes/dummyModel";
 import { verificationSetup } from "./nodes/verificationSetup";
-import { dummyRagasMetrics } from "./nodes/dummyRagasMetrics";
+import { ragasMetrics } from "./nodes/ragasMetrics";
 import { produceRanking } from "./nodes/produceRanking";
 import { createModelNode } from "./nodes/model";
 import { loopEndConditional } from "./conditionals/loop_end";
@@ -31,7 +30,7 @@ const agent = new StateGraph(MessagesState)
 
   .addNode(verificationSetup.name, verificationSetup)
   .addNode("verificationModel", verificationModel)
-  .addNode(dummyRagasMetrics.name, dummyRagasMetrics)
+  .addNode(ragasMetrics.name, ragasMetrics)
   .addNode(produceRanking.name, produceRanking)
   
   .addEdge(START, normalizationSetup.name)
@@ -43,9 +42,9 @@ const agent = new StateGraph(MessagesState)
   .addEdge("triggerEventToolNode", "triggerEventModel")
   
   .addEdge(verificationSetup.name, "verificationModel")
-  .addEdge(verificationSetup.name, dummyRagasMetrics.name)
+  .addEdge(verificationSetup.name, ragasMetrics.name)
 
-  .addEdge(dummyRagasMetrics.name, produceRanking.name)
+  .addEdge(ragasMetrics.name, produceRanking.name)
   .addEdge("verificationModel", produceRanking.name)
 
   // @ts-expect-error

agent/nodes/dummyRagasMetrics.ts

@@ -1,11 +0,0 @@
-import { GraphNode } from "@langchain/langgraph";
-import { MessagesState } from "../state";
-import { AIMessage, HumanMessage } from "@langchain/core/messages";
-
-export const dummyRagasMetrics: GraphNode<typeof MessagesState> = async (state) => {
-  //TODO: get ragas metrics
-  
-  return {
-    messages: [ new AIMessage("RAGASSED : " + state.messages.at(-1)?.content)]
-  };
-};

agent/nodes/normalizationSetup.ts

@@ -6,8 +6,6 @@ import { rankFromCSV } from "../tools/clan/retreiveExamples";
 export const normalizationSetup: GraphNode<typeof MessagesState> = async (state) => {
   let similarityResults = await rankFromCSV(state.disinformationTitle)
 
-  console.log(similarityResults)
-
   let messages : BaseMessage[] = similarityResults.map((item) => {
     return new AIMessage(`Original Claim: ${item.rawtext}. \n\n Normalised Claim: ${item.cleantext}`)
   })

agent/nodes/ragasMetrics.ts

@@ -0,0 +1,18 @@
+import { GraphNode } from "@langchain/langgraph";
+import { MessagesState } from "../state";
+import { AIMessage, HumanMessage } from "@langchain/core/messages";
+import { evaluateWithRagas } from "../tools/ragasCall";
+
+export const ragasMetrics: GraphNode<typeof MessagesState> = async (state) => {
+  const question = "A possible trigger event for: " + state.disinformationTitle //Should it be raw, or normalized?
+  const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
+  const contexts = state.proposedTriggerEvent[state.proposedTriggerEventIndex].context?.split("^^^") ?? []
+  
+  console.log(contexts)
+  
+  const results = await evaluateWithRagas({question, answer, contexts})
+  
+  return {
+    messages: [ new AIMessage("RAGAS:" + results.faithfulness)]
+  };
+};

agent/nodes/verificationSetup.ts

@@ -6,8 +6,7 @@ import { rankAndDisplayData } from "../tools/triggerEventTools";
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
   //this is kinda doing two things, but having two nodes for it seems overkill
-  console.log(state.proposedTriggerEvent)
+
-  console.log(state.proposedTriggerEventIndex)
   if (state.proposedTriggerEvent == undefined) {
     logger.warn("No trigger events in memory, parsing")
 

agent/tools/webSearch.ts

@@ -48,7 +48,7 @@ export async function queryScraper(query: string): Promise<string[]> {
         const desc = (item.description ?? "").trim();
         const link = (item.url ?? "").trim();
 
-        return `- ${title}\n  ${desc}\n  ${link}`;
+        return `^^^ ${title}\n  ${desc}\n  ${link}`;
     });
 
     return lines;