Add removing of duplicates from pipeline. Add to sort step. Move score logic to robertaMetrics node.

agent/nodes/robertaMetrics.ts

@@ -8,7 +8,12 @@ export const robertaMetrics: GraphNode<typeof MessagesState> = async (state) =>
   
   const result = await evaluateWithRoberta({answer})
   
+  let score = 0;
+  if (result.validProb > result.invalidProb) {
+    score = 0.7 + ((result.validProb - result.invalidProb)*0.3);
+  }
+  
   return {
-    messages: [ new AIMessage("ROBERTA:" + result)]
+    messages: [ new AIMessage("ROBERTA:" + score)]
   };
 };

agent/nodes/sort.ts

@@ -1,10 +1,14 @@
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState } from "../state";
 import { AIMessage } from "@langchain/core/messages";
+import { removeDuplicates } from "../tools/removeDuplicates";
 export const sort: GraphNode<typeof MessagesState> = async (state) => {
-  //not sure which will be better from API, just do both
-
   let current = state.proposedTriggerEvent;
+
+  // remove duplicates
+  await removeDuplicates(current)
+
+  // not sure which will be better from API, just do both
   current.sort((a, b) => ((b.score as number) ?? 0) - ((a.score as number) ?? 0));