Add ROBERTA classifier ranking PoC, with 77pc off the bat

agent/nodes/produceRanking.ts

@@ -3,7 +3,7 @@ import { MessagesState } from "../state";
 import { BaseMessage } from "@langchain/core/messages";
 
 //TODO: Each of these might need different weights
-const keys = ["CONFIDENCE", "RAGAS", "RELATION"];
+const keys = ["CONFIDENCE", "RELATION", "RAGAS", "ROBERTA"];
 
 const mapping = {
   VERYHIGH: 1.0,
@@ -16,7 +16,7 @@ const mapping = {
 type Priority = keyof typeof mapping;
 
 function mapResponse(value: string | undefined | null): number {
-  if (!value) return 0;
+  if (!value) return 1;
 
   const trimmed = value.trim();
   const num = parseFloat(trimmed);

agent/nodes/robertaMetrics.ts

@@ -0,0 +1,14 @@
+import { GraphNode } from "@langchain/langgraph";
+import { MessagesState } from "../state";
+import { AIMessage } from "@langchain/core/messages";
+import { evaluateWithRoberta } from "../tools/robertaCall";
+
+export const robertaMetrics: GraphNode<typeof MessagesState> = async (state) => {
+  const answer = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
+  
+  const result = await evaluateWithRoberta({answer})
+  
+  return {
+    messages: [ new AIMessage("ROBERTA:" + result)]
+  };
+};

agent/tools/robertaCall.ts

@@ -0,0 +1,22 @@
+import axios from "axios";
+
+export async function evaluateWithRoberta({
+  answer
+}: {
+  answer: string;
+}) {
+  const res = await axios.post("http://localhost:8000/evaluate", {
+    answer
+  });
+  // console.log(res.data)
+  const validProb = res.data["probabilities"][0][0]
+  const invalidProv = res.data["probabilities"][0][1]
+
+  return validProb > invalidProv ? 1 : 0;
+}
+
+// let res = await evaluateWithRoberta({answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
+// console.log(res)
+
+// res = await evaluateWithRoberta({answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
+// console.log(res)

agent/verify.ts

@@ -6,6 +6,7 @@ import { produceRanking } from "./nodes/produceRanking";
 import { createModelNode } from "./nodes/model";
 import { loopEndConditional } from "./conditionals/loop_end";
 import { sort } from "./nodes/sort";
+import { robertaMetrics } from "./nodes/robertaMetrics";
 
 const verificationModel = createModelNode([], "verify.txt");
 const relationModel = createModelNode([], "relation.txt");
@@ -14,21 +15,24 @@ const agent = new StateGraph(MessagesState)
   
   //NODES
   .addNode(verificationSetup.name, verificationSetup)
-  .addNode("verificationModel", verificationModel)
-  .addNode(ragasMetrics.name, ragasMetrics)
-  .addNode("relationModel", relationModel)
+  // .addNode("verificationModel", verificationModel)
+  // .addNode(ragasMetrics.name, ragasMetrics)
+  .addNode(robertaMetrics.name, robertaMetrics)
+  // .addNode("relationModel", relationModel)
   
   .addNode(produceRanking.name, produceRanking)
   .addNode(sort.name, sort)
   
   .addEdge(START, verificationSetup.name)
-  .addEdge(verificationSetup.name, "verificationModel")
-  .addEdge(verificationSetup.name, ragasMetrics.name)
-  .addEdge(verificationSetup.name, "relationModel")
+  // .addEdge(verificationSetup.name, "verificationModel")
+  // .addEdge(verificationSetup.name, ragasMetrics.name)
+  .addEdge(verificationSetup.name, robertaMetrics.name)
+  // .addEdge(verificationSetup.name, "relationModel")
 
-  .addEdge(ragasMetrics.name, produceRanking.name)
-  .addEdge("verificationModel", produceRanking.name)
-  .addEdge("relationModel", produceRanking.name)
+  // .addEdge(ragasMetrics.name, produceRanking.name)
+  .addEdge(robertaMetrics.name, produceRanking.name)
+  // .addEdge("verificationModel", produceRanking.name)
+  // .addEdge("relationModel", produceRanking.name)
 
   // @ts-expect-error
   .addConditionalEdges(produceRanking.name, loopEndConditional, [verificationSetup.name, sort.name])