implement verification model

agent/agent.ts

@@ -13,11 +13,9 @@ import { loopEndConditional } from "./conditionals/loop_end";
 
 const triggerEventToolNode = createToolNode(triggerEventToolsByName);
 
-const dummyVerificationModel = createDummyModelNode("verification of");
-
 const normalisationModel = createModelNode([], "normalization.txt");
 const triggerEventModel = createModelNode(triggerEventToolsByName, "trigger.txt");
-
+const verificationModel = createModelNode([], "verify.txt");
 
 const triggerEventToolConditional = createToolConditional("triggerEventToolNode", verificationSetup.name);
 
@@ -32,7 +30,7 @@ const agent = new StateGraph(MessagesState)
   .addNode("triggerEventModel", triggerEventModel)
 
   .addNode(verificationSetup.name, verificationSetup)
-  .addNode("dummyVerificationModel", dummyVerificationModel)
+  .addNode("verificationModel", verificationModel)
   .addNode(dummyRagasMetrics.name, dummyRagasMetrics)
   .addNode(produceRanking.name, produceRanking)
   
@@ -44,13 +42,13 @@ const agent = new StateGraph(MessagesState)
   .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", verificationSetup.name])
   .addEdge("triggerEventToolNode", "triggerEventModel")
   
-  .addEdge(verificationSetup.name, "dummyVerificationModel")
+  .addEdge(verificationSetup.name, "verificationModel")
   .addEdge(verificationSetup.name, dummyRagasMetrics.name)
 
   .addEdge(dummyRagasMetrics.name, produceRanking.name)
-  .addEdge("dummyVerificationModel", produceRanking.name)
+  .addEdge("verificationModel", produceRanking.name)
 
-  
+  // @ts-expect-error
   .addConditionalEdges(produceRanking.name, loopEndConditional, [verificationSetup.name, END])
   
   .compile();

agent/nodes/model.ts

@@ -6,7 +6,7 @@ import { hydratePrompt } from "../prompts/hydratePrompt";
 
 export function createModelNode(tools: any, promptPath: string): GraphNode<typeof MessagesState> {
     return async (state) => {
-        const sysPrompt = hydratePrompt(promptPath, state);
+        const sysPrompt = await hydratePrompt(promptPath, state);
 
         const model = new ChatOpenAI({
             model: "gpt-5-mini"

agent/nodes/verificationSetup.ts

@@ -1,6 +1,8 @@
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState, ProposedTriggerEventArray } from "../state";
 import { logger } from "../utils/logger";
+import { queryScraper } from "../tools/webSearch";
+import { rankAndDisplayData } from "../tools/triggerEventTools";
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
   //this is kinda doing two things, but having two nodes for it seems overkill
@@ -12,6 +14,14 @@ export const verificationSetup: GraphNode<typeof MessagesState> = async (state)
     let genResponse = state.messages.at(-1)?.content.toString() ?? "";
     const parsed = ProposedTriggerEventArray.parse(JSON.parse(genResponse));
 
+    for (let i = 0; i < parsed.length; i++) {
+      const search = parsed[i].SearchQuery
+      const data = await queryScraper(search);
+      const output = await rankAndDisplayData(data, search);
+
+      parsed[i].context = output;
+    }
+    
     return { proposedTriggerEvent: parsed, proposedTriggerEventIndex: 0 };
   }
   else {

agent/prompts/hydratePrompt.ts

@@ -1,6 +1,8 @@
 import fs from "fs";
+import { queryScraper } from "../tools/webSearch";
+import { rankAndDisplayData } from "../tools/triggerEventTools";
 
-export function hydratePrompt(path: string, state: any) {
+export async function hydratePrompt(path: string, state: any) : Promise<string> {
     // TODO: expand into full context-based replacement engine
 
     let raw = fs.readFileSync("prompts/" + path, "utf-8");
@@ -8,5 +10,15 @@ export function hydratePrompt(path: string, state: any) {
     raw = raw.replace("###TITLE###", state.disinformationTitle);
     raw = raw.replace("###LM###", state.messages.at(-1).content);
 
+    if (raw.indexOf("###TECLAIM###") != -1) {
+        const title = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event
+        raw = raw.replace("###TECLAIM###", title)
+    }
+
+    if (raw.indexOf("###TESEARCH###") != -1) {
+        const output = state.proposedTriggerEvent[state.proposedTriggerEventIndex].context
+        raw = raw.replace("###TESEARCH###", output)
+    }
+
     return raw;
 }

agent/prompts/verify.txt

@@ -0,0 +1,8 @@
+Do the search results cited below
+###TESEARCH###
+Support the idea that the following happened:
+###TECLAIM###
+
+Respond with "CONFIDENCE", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
+
+Dates can be off by a few days, that would still be valid

agent/state.ts

@@ -15,7 +15,8 @@ export const ProposedTriggerEvent = z.object({
   ReasoningWhyRelevant: z.string(),
   SearchQuery: z.string(),
   Url: z.url(),
-  IsItselfDisinformation: z.boolean()
+  IsItselfDisinformation: z.boolean(),
+  context: z.string().optional()
 })
 
 export const ProposedTriggerEventArray = z.array(ProposedTriggerEvent);

agent/tools/triggerEventTools.ts

@@ -5,7 +5,7 @@ import { extractWebpageContent } from "./webpageFetch";
 import { rankDynamically } from "./clan/retreiveExamples";
 
 
-async function rankAndDisplayData(data: string[], context: string):Promise<string> {
+export async function rankAndDisplayData(data: string[], context: string):Promise<string> {
   let index = 0;
   let ranked = await rankDynamically(context, data.map(irm => ({ id: index++, rawtext: irm })))
   return ranked.map(itm => itm.rawtext).join("\n")