Implement ensemble into final model structure

agent/nodes/ensembleNode.ts

@@ -0,0 +1,17 @@
+import { GraphNode } from "@langchain/langgraph";
+import { MessagesState } from "../state";
+import { AIMessage } from "@langchain/core/messages";
+import { evaluateWithEnsemble } from "../tools/ensembleCall";
+
+export function createEnsembleNode(title: string, method: string): GraphNode<typeof MessagesState> {
+  return async (state) => {
+    const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
+
+    const result = await evaluateWithEnsemble({ answer, method })
+    const score = result.validProb - result.invalidProb;
+
+    return {
+      messages: [new AIMessage(title + ":" + score)]
+    };
+  };
+};

agent/nodes/produceRanking.ts

@@ -2,31 +2,25 @@ import { GraphNode } from "@langchain/langgraph";
 import { MessagesState } from "../state";
 import { BaseMessage } from "@langchain/core/messages";
 
-//TODO: Each of these might need different weights
-const keys = ["CONFIDENCE", "RELATION", "RAGAS", "ROBERTA"];
-
-const mapping = {
-  VERYHIGH: 1.0,
-  HIGH: 0.75,
-  MEDIUM: 0.5,
-  LOW: 0.25,
-  VERYLOW: 0.0,
+const models = {
+  REGRESSION: 0.3,
+  ROBERTA: 0.5,
+  FLAN: 0.3,
 } as const;
 
-type Priority = keyof typeof mapping;
+type ModelKey = keyof typeof models;
 
 function mapResponse(value: string | undefined | null): number {
-  if (!value) return 1;
+  if (!value) return 0;
 
   const trimmed = value.trim();
   const num = parseFloat(trimmed);
 
-  // If number, return it
-  if (!isNaN(num)) return num;
-
-  // Otherwise, map to value
-  const upper = trimmed.toUpperCase() as Priority;
-  return mapping[upper] ?? 0;
+  if (!isNaN(num)) {
+    return num;
+  } else {
+    return 0;
+  }
 }
 
 function getLastMessageContaining(
@@ -43,18 +37,18 @@ function getLastMessageContaining(
 }
 
 export const produceRanking: GraphNode<typeof MessagesState> = async (state) => {
-  // Extract and map values
-  const values = keys.map((key) => {
+  const values = (Object.keys(models) as ModelKey[]).map((key) => {
     const msg = getLastMessageContaining(state.messages, key);
     const part = msg?.split(":").at(1);
-    return mapResponse(part);
+    const baseValue = mapResponse(part);
+
+    return baseValue * models[key];
   });
 
-  // Multiply!
-  const result = values.reduce((acc, val) => acc * val, 1);
+  const result = values.reduce((acc, val) => acc + val, 0);
 
   const current = state.proposedTriggerEvent;
   current[state.proposedTriggerEventIndex].score = result;
 
   return { proposedTriggerEvent: current };
-};
+};

agent/nodes/ragasMetrics.ts

@@ -1,16 +0,0 @@
-import { GraphNode } from "@langchain/langgraph";
-import { MessagesState } from "../state";
-import { AIMessage, HumanMessage } from "@langchain/core/messages";
-import { evaluateWithRagas } from "../tools/ragasCall";
-
-export const ragasMetrics: GraphNode<typeof MessagesState> = async (state) => {
-  const question = "A possible trigger event for: " + state.disinformationTitle //Should it be raw, or normalized?
-  const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
-  const contexts = state.proposedTriggerEvent[state.proposedTriggerEventIndex].context?.split("^^^") ?? []
-  
-  const results = await evaluateWithRagas({question, answer, contexts})
-  
-  return {
-    messages: [ new AIMessage("RAGAS:" + results.faithfulness)]
-  };
-};

agent/nodes/robertaMetrics.ts

@@ -1,39 +0,0 @@
-import { GraphNode } from "@langchain/langgraph";
-import { MessagesState } from "../state";
-import { AIMessage } from "@langchain/core/messages";
-import { evaluateWithRoberta } from "../tools/robertaCall";
-
-export const robertaMetrics: GraphNode<typeof MessagesState> = async (state) => {
-  const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
-  
-  const lrresult = await evaluateWithRoberta({answer, method:"logreg"})
-  const lrscore = lrresult.validProb - lrresult.invalidProb;
-  
-  const roresult = await evaluateWithRoberta({answer, method:"roberta"})
-  const roscore = roresult.validProb - roresult.invalidProb;
-  
-  const flresult = await evaluateWithRoberta({answer, method:"flan"})
-  const flscore = flresult.validProb - flresult.invalidProb;
-  
-  //Option 1: combining scores
-  const score = lrscore * 0.3 + roscore * 0.5 + flscore * 0.3
-
-  //Option 2: majority voting
-  // const rovote = roscore > 0.6
-  // const flvote = flscore > 0.94
-  // const lrvote = lrscore > 0.75
-
-  // let counter = 0
-  // if (rovote) counter++
-  // if (flvote) counter++
-  // if (lrvote) counter++
-
-  // let score = 0
-  // if (counter >= 2) {
-  //   score = 0.7 + lrscore + flscore + lrscore
-  // }
-  
-  return {
-    messages: [ new AIMessage("ROBERTA:" + score)]
-  };
-};