From 6dd6bf7eaf3148c1e9a8fb7f354e4862c95fa091 Mon Sep 17 00:00:00 2001 From: William Jeynes Date: Thu, 12 Feb 2026 22:32:24 +0000 Subject: [PATCH] implement verification model --- agent/agent.ts | 12 +++++------- agent/nodes/model.ts | 2 +- agent/nodes/verificationSetup.ts | 10 ++++++++++ agent/prompts/hydratePrompt.ts | 14 +++++++++++++- agent/prompts/verify.txt | 8 ++++++++ agent/state.ts | 3 ++- agent/tools/triggerEventTools.ts | 2 +- 7 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 agent/prompts/verify.txt diff --git a/agent/agent.ts b/agent/agent.ts index 75bc22d..6b0e4b7 100644 --- a/agent/agent.ts +++ b/agent/agent.ts @@ -13,11 +13,9 @@ import { loopEndConditional } from "./conditionals/loop_end"; const triggerEventToolNode = createToolNode(triggerEventToolsByName); -const dummyVerificationModel = createDummyModelNode("verification of"); - const normalisationModel = createModelNode([], "normalization.txt"); const triggerEventModel = createModelNode(triggerEventToolsByName, "trigger.txt"); - +const verificationModel = createModelNode([], "verify.txt"); const triggerEventToolConditional = createToolConditional("triggerEventToolNode", verificationSetup.name); @@ -32,7 +30,7 @@ const agent = new StateGraph(MessagesState) .addNode("triggerEventModel", triggerEventModel) .addNode(verificationSetup.name, verificationSetup) - .addNode("dummyVerificationModel", dummyVerificationModel) + .addNode("verificationModel", verificationModel) .addNode(dummyRagasMetrics.name, dummyRagasMetrics) .addNode(produceRanking.name, produceRanking) @@ -44,13 +42,13 @@ const agent = new StateGraph(MessagesState) .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", verificationSetup.name]) .addEdge("triggerEventToolNode", "triggerEventModel") - .addEdge(verificationSetup.name, "dummyVerificationModel") + .addEdge(verificationSetup.name, "verificationModel") .addEdge(verificationSetup.name, dummyRagasMetrics.name) .addEdge(dummyRagasMetrics.name, produceRanking.name) - .addEdge("dummyVerificationModel", produceRanking.name) + .addEdge("verificationModel", produceRanking.name) - + // @ts-expect-error .addConditionalEdges(produceRanking.name, loopEndConditional, [verificationSetup.name, END]) .compile(); diff --git a/agent/nodes/model.ts b/agent/nodes/model.ts index 008d830..a251e1d 100644 --- a/agent/nodes/model.ts +++ b/agent/nodes/model.ts @@ -6,7 +6,7 @@ import { hydratePrompt } from "../prompts/hydratePrompt"; export function createModelNode(tools: any, promptPath: string): GraphNode { return async (state) => { - const sysPrompt = hydratePrompt(promptPath, state); + const sysPrompt = await hydratePrompt(promptPath, state); const model = new ChatOpenAI({ model: "gpt-5-mini" diff --git a/agent/nodes/verificationSetup.ts b/agent/nodes/verificationSetup.ts index 7c20518..364365c 100644 --- a/agent/nodes/verificationSetup.ts +++ b/agent/nodes/verificationSetup.ts @@ -1,6 +1,8 @@ import { GraphNode } from "@langchain/langgraph"; import { MessagesState, ProposedTriggerEventArray } from "../state"; import { logger } from "../utils/logger"; +import { queryScraper } from "../tools/webSearch"; +import { rankAndDisplayData } from "../tools/triggerEventTools"; export const verificationSetup: GraphNode = async (state) => { //this is kinda doing two things, but having two nodes for it seems overkill @@ -12,6 +14,14 @@ export const verificationSetup: GraphNode = async (state) let genResponse = state.messages.at(-1)?.content.toString() ?? ""; const parsed = ProposedTriggerEventArray.parse(JSON.parse(genResponse)); + for (let i = 0; i < parsed.length; i++) { + const search = parsed[i].SearchQuery + const data = await queryScraper(search); + const output = await rankAndDisplayData(data, search); + + parsed[i].context = output; + } + return { proposedTriggerEvent: parsed, proposedTriggerEventIndex: 0 }; } else { diff --git a/agent/prompts/hydratePrompt.ts b/agent/prompts/hydratePrompt.ts index dbf4b21..555a42a 100644 --- a/agent/prompts/hydratePrompt.ts +++ b/agent/prompts/hydratePrompt.ts @@ -1,6 +1,8 @@ import fs from "fs"; +import { queryScraper } from "../tools/webSearch"; +import { rankAndDisplayData } from "../tools/triggerEventTools"; -export function hydratePrompt(path: string, state: any) { +export async function hydratePrompt(path: string, state: any) : Promise { // TODO: expand into full context-based replacement engine let raw = fs.readFileSync("prompts/" + path, "utf-8"); @@ -8,5 +10,15 @@ export function hydratePrompt(path: string, state: any) { raw = raw.replace("###TITLE###", state.disinformationTitle); raw = raw.replace("###LM###", state.messages.at(-1).content); + if (raw.indexOf("###TECLAIM###") != -1) { + const title = state.proposedTriggerEvent[state.proposedTriggerEventIndex].Event + raw = raw.replace("###TECLAIM###", title) + } + + if (raw.indexOf("###TESEARCH###") != -1) { + const output = state.proposedTriggerEvent[state.proposedTriggerEventIndex].context + raw = raw.replace("###TESEARCH###", output) + } + return raw; } diff --git a/agent/prompts/verify.txt b/agent/prompts/verify.txt new file mode 100644 index 0000000..3df5d86 --- /dev/null +++ b/agent/prompts/verify.txt @@ -0,0 +1,8 @@ +Do the search results cited below +###TESEARCH### +Support the idea that the following happened: +###TECLAIM### + +Respond with "CONFIDENCE", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format. + +Dates can be off by a few days, that would still be valid \ No newline at end of file diff --git a/agent/state.ts b/agent/state.ts index 14ef927..d436243 100644 --- a/agent/state.ts +++ b/agent/state.ts @@ -15,7 +15,8 @@ export const ProposedTriggerEvent = z.object({ ReasoningWhyRelevant: z.string(), SearchQuery: z.string(), Url: z.url(), - IsItselfDisinformation: z.boolean() + IsItselfDisinformation: z.boolean(), + context: z.string().optional() }) export const ProposedTriggerEventArray = z.array(ProposedTriggerEvent); diff --git a/agent/tools/triggerEventTools.ts b/agent/tools/triggerEventTools.ts index 6d7c0c4..773e2be 100644 --- a/agent/tools/triggerEventTools.ts +++ b/agent/tools/triggerEventTools.ts @@ -5,7 +5,7 @@ import { extractWebpageContent } from "./webpageFetch"; import { rankDynamically } from "./clan/retreiveExamples"; -async function rankAndDisplayData(data: string[], context: string):Promise { +export async function rankAndDisplayData(data: string[], context: string):Promise { let index = 0; let ranked = await rankDynamically(context, data.map(irm => ({ id: index++, rawtext: irm }))) return ranked.map(itm => itm.rawtext).join("\n")