diff --git a/agent/agent.ts b/agent/agent.ts index c4be29d..475bfce 100644 --- a/agent/agent.ts +++ b/agent/agent.ts @@ -15,13 +15,13 @@ const triggerEventToolNode = createToolNode(triggerEventToolsByName); const normalisationModel = createModelNode([], "normalization.txt"); const triggerEventModel = createModelNode(triggerEventToolsByName, "trigger.txt"); const verificationModel = createModelNode([], "verify.txt"); +const relationModel = createModelNode([], "relation.txt"); const triggerEventToolConditional = createToolConditional("triggerEventToolNode", verificationSetup.name); const agent = new StateGraph(MessagesState) //NODES - .addNode(normalizationSetup.name, normalizationSetup) .addNode("normalisationModel", normalisationModel) @@ -31,6 +31,8 @@ const agent = new StateGraph(MessagesState) .addNode(verificationSetup.name, verificationSetup) .addNode("verificationModel", verificationModel) .addNode(ragasMetrics.name, ragasMetrics) + .addNode("relationModel", relationModel) + .addNode(produceRanking.name, produceRanking) .addEdge(START, normalizationSetup.name) @@ -43,9 +45,11 @@ const agent = new StateGraph(MessagesState) .addEdge(verificationSetup.name, "verificationModel") .addEdge(verificationSetup.name, ragasMetrics.name) + .addEdge(verificationSetup.name, "relationModel") .addEdge(ragasMetrics.name, produceRanking.name) .addEdge("verificationModel", produceRanking.name) + .addEdge("relationModel", produceRanking.name) // @ts-expect-error .addConditionalEdges(produceRanking.name, loopEndConditional, [verificationSetup.name, END]) diff --git a/agent/nodes/dummyModel.ts b/agent/nodes/dummyModel.ts deleted file mode 100644 index 299ecc9..0000000 --- a/agent/nodes/dummyModel.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { GraphNode } from "@langchain/langgraph"; -import { MessagesState } from "../state"; -import { AIMessage } from "@langchain/core/messages"; - -export function createDummyModelNode(addition): GraphNode { - return async (state) => { - //TODO: call AI model with collected data - - return { - messages: [new AIMessage(addition + " : " + state.messages.at(-1)?.content)] - }; - }; -} \ No newline at end of file diff --git a/agent/nodes/produceRanking.ts b/agent/nodes/produceRanking.ts index 5665934..500abf6 100644 --- a/agent/nodes/produceRanking.ts +++ b/agent/nodes/produceRanking.ts @@ -1,9 +1,50 @@ import { GraphNode } from "@langchain/langgraph"; import { MessagesState } from "../state"; -import { AIMessage, HumanMessage } from "@langchain/core/messages"; +import { BaseMessage } from "@langchain/core/messages"; + +type Priority = keyof typeof mapping; + +const mapping = { + VERYHIGH: 1.0, + HIGH: 0.75, + MEDIUM: 0.5, + LOW: 0.25, + VERYLOW: 0.0 +} as const; + +function mapResponse(value: string): number { + const upper = value.toUpperCase() as Priority; + + if (upper in mapping) { + return mapping[upper]; + } + + return 0; +} + +function getLastMessageContaining( + messages: BaseMessage[], + searchString: string +): string { + for (let i = messages.length - 1; i >= 0; i--) { + const content = messages[i].content; + if (typeof content === "string" && content.includes(searchString)) { + return content; + } + } + return ""; +} export const produceRanking: GraphNode = async (state) => { - //TODO: produce ranking here + //TODO: what should these weights be + let conf = getLastMessageContaining(state.messages, "CONFIDENCE")?.split(":")[1] //TODO: we can better error handle here + let ragas = getLastMessageContaining(state.messages, "RAGAS")?.split(":")[1] //TODO: we can genericify this too surely + let rel = getLastMessageContaining(state.messages, "RELATION")?.split(":")[1] + + let result = mapResponse(conf) * Number.parseFloat(ragas) * mapResponse(rel) - return { messages: [ new AIMessage(state.messages?.length.toString() ?? "0")] }; + let current = state.proposedTriggerEvent; + current[state.proposedTriggerEventIndex].score = result; + + return { proposedTriggerEvent: current }; }; \ No newline at end of file diff --git a/agent/prompts/relation.txt b/agent/prompts/relation.txt new file mode 100644 index 0000000..9136212 --- /dev/null +++ b/agent/prompts/relation.txt @@ -0,0 +1,9 @@ +Could the following real-world event: +###TECLAIM### + +Be a trigger for the following disinformation: +###TITLE### + +Respond with "RELATION", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format. + +Ignore wether the event happened or not, purely consider the likiness of causation diff --git a/agent/state.ts b/agent/state.ts index d436243..d9e5cb1 100644 --- a/agent/state.ts +++ b/agent/state.ts @@ -16,7 +16,8 @@ export const ProposedTriggerEvent = z.object({ SearchQuery: z.string(), Url: z.url(), IsItselfDisinformation: z.boolean(), - context: z.string().optional() + context: z.string().optional(), + score: z.number().optional }) export const ProposedTriggerEventArray = z.array(ProposedTriggerEvent);