Implement ensemble into final model structure

agent/agent.ts

@@ -10,7 +10,7 @@ import { createModelNode } from "./nodes/model";
 import { loopEndConditional } from "./conditionals/loop_end";
 import { sort } from "./nodes/sort";
 import { triggerEventSetup } from "./nodes/triggerEventSetup";
-import { robertaMetrics } from "./nodes/robertaMetrics";
+import { createEnsembleNode } from "./nodes/ensembleNode";
 
 const triggerEventToolNode = createToolNode(triggerEventToolsByName);
 
@@ -19,6 +19,10 @@ const triggerEventModel = createModelNode(triggerEventToolsByName, "trigger.txt"
 
 const triggerEventToolConditional = createToolConditional("triggerEventToolNode", verificationSetup.name);
 
+const roNode = createEnsembleNode("ROBERTA", "roberta");
+const flNode = createEnsembleNode("FLAN", "flan");
+const lrNode = createEnsembleNode("REGRESSION", "logreg");
+
 const agent = new StateGraph(MessagesState)
   
   //NODES
@@ -30,7 +34,10 @@ const agent = new StateGraph(MessagesState)
   .addNode("triggerEventModel", triggerEventModel)
 
   .addNode(verificationSetup.name, verificationSetup)
-  .addNode(robertaMetrics.name, robertaMetrics)
+
+  .addNode("roNode", roNode)
+  .addNode("flNode", flNode)
+  .addNode("lrNode", lrNode)
   
   .addNode(produceRanking.name, produceRanking)
   .addNode(sort.name, sort)
@@ -45,9 +52,13 @@ const agent = new StateGraph(MessagesState)
   .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", verificationSetup.name])
   .addEdge("triggerEventToolNode", "triggerEventModel")
   
-  .addEdge(verificationSetup.name, robertaMetrics.name)
+  .addEdge(verificationSetup.name, "roNode")
+  .addEdge(verificationSetup.name, "flNode")
+  .addEdge(verificationSetup.name, "lrNode")
   
-  .addEdge(robertaMetrics.name, produceRanking.name)
+  .addEdge("roNode", produceRanking.name)
+  .addEdge("flNode", produceRanking.name)
+  .addEdge("lrNode", produceRanking.name)
 
   // @ts-expect-error
   .addConditionalEdges(produceRanking.name, loopEndConditional, [verificationSetup.name, sort.name])

agent/nodes/ensembleNode.ts

@@ -0,0 +1,17 @@
+import { GraphNode } from "@langchain/langgraph";
+import { MessagesState } from "../state";
+import { AIMessage } from "@langchain/core/messages";
+import { evaluateWithEnsemble } from "../tools/ensembleCall";
+
+export function createEnsembleNode(title: string, method: string): GraphNode<typeof MessagesState> {
+  return async (state) => {
+    const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
+
+    const result = await evaluateWithEnsemble({ answer, method })
+    const score = result.validProb - result.invalidProb;
+
+    return {
+      messages: [new AIMessage(title + ":" + score)]
+    };
+  };
+};

agent/nodes/produceRanking.ts

@@ -2,31 +2,25 @@ import { GraphNode } from "@langchain/langgraph";
 import { MessagesState } from "../state";
 import { BaseMessage } from "@langchain/core/messages";
 
-//TODO: Each of these might need different weights
-const keys = ["CONFIDENCE", "RELATION", "RAGAS", "ROBERTA"];
-
-const mapping = {
-  VERYHIGH: 1.0,
-  HIGH: 0.75,
-  MEDIUM: 0.5,
-  LOW: 0.25,
-  VERYLOW: 0.0,
+const models = {
+  REGRESSION: 0.3,
+  ROBERTA: 0.5,
+  FLAN: 0.3,
 } as const;
 
-type Priority = keyof typeof mapping;
+type ModelKey = keyof typeof models;
 
 function mapResponse(value: string | undefined | null): number {
-  if (!value) return 1;
+  if (!value) return 0;
 
   const trimmed = value.trim();
   const num = parseFloat(trimmed);
 
-  // If number, return it
-  if (!isNaN(num)) return num;
-
-  // Otherwise, map to value
-  const upper = trimmed.toUpperCase() as Priority;
-  return mapping[upper] ?? 0;
+  if (!isNaN(num)) {
+    return num;
+  } else {
+    return 0;
+  }
 }
 
 function getLastMessageContaining(
@@ -43,18 +37,18 @@ function getLastMessageContaining(
 }
 
 export const produceRanking: GraphNode<typeof MessagesState> = async (state) => {
-  // Extract and map values
-  const values = keys.map((key) => {
+  const values = (Object.keys(models) as ModelKey[]).map((key) => {
     const msg = getLastMessageContaining(state.messages, key);
     const part = msg?.split(":").at(1);
-    return mapResponse(part);
+    const baseValue = mapResponse(part);
+
+    return baseValue * models[key];
   });
 
-  // Multiply!
-  const result = values.reduce((acc, val) => acc * val, 1);
+  const result = values.reduce((acc, val) => acc + val, 0);
 
   const current = state.proposedTriggerEvent;
   current[state.proposedTriggerEventIndex].score = result;
 
   return { proposedTriggerEvent: current };
-};
+};

agent/nodes/ragasMetrics.ts

@@ -1,16 +0,0 @@
-import { GraphNode } from "@langchain/langgraph";
-import { MessagesState } from "../state";
-import { AIMessage, HumanMessage } from "@langchain/core/messages";
-import { evaluateWithRagas } from "../tools/ragasCall";
-
-export const ragasMetrics: GraphNode<typeof MessagesState> = async (state) => {
-  const question = "A possible trigger event for: " + state.disinformationTitle //Should it be raw, or normalized?
-  const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
-  const contexts = state.proposedTriggerEvent[state.proposedTriggerEventIndex].context?.split("^^^") ?? []
-  
-  const results = await evaluateWithRagas({question, answer, contexts})
-  
-  return {
-    messages: [ new AIMessage("RAGAS:" + results.faithfulness)]
-  };
-};

agent/nodes/robertaMetrics.ts

@@ -1,39 +0,0 @@
-import { GraphNode } from "@langchain/langgraph";
-import { MessagesState } from "../state";
-import { AIMessage } from "@langchain/core/messages";
-import { evaluateWithRoberta } from "../tools/robertaCall";
-
-export const robertaMetrics: GraphNode<typeof MessagesState> = async (state) => {
-  const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
-  
-  const lrresult = await evaluateWithRoberta({answer, method:"logreg"})
-  const lrscore = lrresult.validProb - lrresult.invalidProb;
-  
-  const roresult = await evaluateWithRoberta({answer, method:"roberta"})
-  const roscore = roresult.validProb - roresult.invalidProb;
-  
-  const flresult = await evaluateWithRoberta({answer, method:"flan"})
-  const flscore = flresult.validProb - flresult.invalidProb;
-  
-  //Option 1: combining scores
-  const score = lrscore * 0.3 + roscore * 0.5 + flscore * 0.3
-
-  //Option 2: majority voting
-  // const rovote = roscore > 0.6
-  // const flvote = flscore > 0.94
-  // const lrvote = lrscore > 0.75
-
-  // let counter = 0
-  // if (rovote) counter++
-  // if (flvote) counter++
-  // if (lrvote) counter++
-
-  // let score = 0
-  // if (counter >= 2) {
-  //   score = 0.7 + lrscore + flscore + lrscore
-  // }
-  
-  return {
-    messages: [ new AIMessage("ROBERTA:" + score)]
-  };
-};

agent/tools/ensembleCall.ts

@@ -1,6 +1,6 @@
 import axios from "axios";
 
-export async function evaluateWithRoberta({
+export async function evaluateWithEnsemble({
   answer,
   method
 }: {
@@ -10,7 +10,7 @@ export async function evaluateWithRoberta({
   const res = await axios.post("http://localhost:8000/evaluate", {
     answer,
     method
-  });
+  }, {timeout: 0});
   // console.log(res.data)
   const validProb = res.data["probabilities"][0][0]
   const invalidProb = res.data["probabilities"][0][1] + res.data["probabilities"][0][2]

agent/verify.ts

@@ -1,39 +1,36 @@
 import { END, START, StateGraph } from "@langchain/langgraph";
 import { MessagesState } from "./state";
 import { verificationSetup } from "./nodes/verificationSetup";
-import { ragasMetrics } from "./nodes/ragasMetrics";
 import { produceRanking } from "./nodes/produceRanking";
-import { createModelNode } from "./nodes/model";
 import { loopEndConditional } from "./conditionals/loop_end";
 import { sort } from "./nodes/sort";
-import { robertaMetrics } from "./nodes/robertaMetrics";
+import { createEnsembleNode } from "./nodes/ensembleNode";
 
-const verificationModel = createModelNode([], "verify.txt");
-const relationModel = createModelNode([], "relation.txt");
+const roNode = createEnsembleNode("ROBERTA", "roberta");
+const flNode = createEnsembleNode("FLAN", "flan");
+const lrNode = createEnsembleNode("REGRESSION", "logreg");
 
 const agent = new StateGraph(MessagesState)
   
   //NODES
   .addNode(verificationSetup.name, verificationSetup)
-  // .addNode("verificationModel", verificationModel)
-  // .addNode(ragasMetrics.name, ragasMetrics)
-  .addNode(robertaMetrics.name, robertaMetrics)
-  // .addNode("relationModel", relationModel)
+  .addNode("roNode", roNode)
+  .addNode("flNode", flNode)
+  .addNode("lrNode", lrNode)
   
   .addNode(produceRanking.name, produceRanking)
   .addNode(sort.name, sort)
   
   .addEdge(START, verificationSetup.name)
-  // .addEdge(verificationSetup.name, "verificationModel")
-  // .addEdge(verificationSetup.name, ragasMetrics.name)
-  .addEdge(verificationSetup.name, robertaMetrics.name)
-  // .addEdge(verificationSetup.name, "relationModel")
-
-  // .addEdge(ragasMetrics.name, produceRanking.name)
-  .addEdge(robertaMetrics.name, produceRanking.name)
-  // .addEdge("verificationModel", produceRanking.name)
-  // .addEdge("relationModel", produceRanking.name)
+  
+  .addEdge(verificationSetup.name, "roNode")
+  .addEdge(verificationSetup.name, "flNode")
+  .addEdge(verificationSetup.name, "lrNode")
 
+  .addEdge("roNode", produceRanking.name)
+  .addEdge("flNode", produceRanking.name)
+  .addEdge("lrNode", produceRanking.name)
+  
   // @ts-expect-error
   .addConditionalEdges(produceRanking.name, loopEndConditional, [verificationSetup.name, sort.name])