make prompt worse

agent/.env.example

@@ -4,4 +4,3 @@ LANGSMITH_API_KEY=123456
 LANGSMITH_ENDPOINT=https://eu.api.smith.langchain.com
 SCRAPER_INSTANCE=https://example.com
 SCRAPER_PARAM_ANYTHING=else
-RANKING_URL=http://localhost:8000/evaluate

agent/nodes/model.ts

@@ -1,28 +1,25 @@
-import { SystemMessage } from "@langchain/core/messages";
+import { HumanMessage, SystemMessage } from "@langchain/core/messages";
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState } from "../state";
-import { ChatOllama } from "@langchain/ollama";
+import { ChatOpenAI } from "@langchain/openai"
 import { hydratePrompt } from "../prompts/hydratePrompt";
-import { logger } from "../utils/logger";
 
 export function createModelNode(tools: any, promptPath: string): GraphNode<typeof MessagesState> {
     return async (state) => {
         const sysPrompt = await hydratePrompt(promptPath, state);
 
-        const model = new ChatOllama({
+        const model = new ChatOpenAI({
-            model: "llama3.1:8b-instruct-q4_K_M",
+            model: "gpt-5-mini"
-            temperature: 0.3
         });
-
         const modelWithTools = model.bindTools(Object.values(tools));
 
         const response = await modelWithTools.invoke([
-            new SystemMessage(sysPrompt),
+            new SystemMessage(
+                sysPrompt
+            ),
             ...state.messages,
         ]);
 
-        logger.error(response);
-
         return {
             messages: [response]
         };

agent/nodes/triggerEventSetup.ts

@@ -3,16 +3,8 @@ import { MessagesState } from "../state";
 import { AIMessage, BaseMessage } from "@langchain/core/messages";
 import { rankExampleTriggerEvents } from "../tools/retreiveExamples";
 
-function extractTE(text: string) {
-  const match = text.match(/<norm>([\s\S]*?)<\/norm>/);
-  if (!match) throw new Error("Nothing found between <norm> tags");
-  return match[1].trim();
-}
-
-
 export const triggerEventSetup: GraphNode<typeof MessagesState> = async (state) => {
-  let raw = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
+  let nc = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
-  let nc = extractTE(raw.toString())
   
   //Now give in-context examples. hopwfully we can self-teach?
   let similarityResults = await rankExampleTriggerEvents(state.disinformationTitle)

agent/nodes/verificationSetup.ts

@@ -1,60 +1,32 @@
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState, ProposedTriggerEventArray } from "../state";
 import { logger } from "../utils/logger";
-import { jsonrepair } from 'jsonrepair';
+import { queryScraper } from "../tools/webSearch";
-
+import { rankAndDisplayData } from "../tools/triggerEventTools";
-function extractJSON(text: string) {
-  const match = text.match(/<json>([\s\S]*?)<\/json>/);
-  if (!match) throw new Error("No JSON found between <json> tags");
-  return match[1].trim();
-}
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
+  //this is kinda doing two things, but having two nodes for it seems overkill
+
   if (state.proposedTriggerEvent == undefined) {
-    logger.warn("No trigger events in memory, parsing");
+    logger.warn("No trigger events in memory, parsing")
 
-    const genResponse = state.messages.at(-1)?.content.toString() ?? "";
+    let genResponse = state.messages.at(-1)?.content.toString() ?? "";
+    const parsed = ProposedTriggerEventArray.parse(JSON.parse(genResponse));
 
-    let repaired: string;
+    for (let i = 0; i < parsed.length; i++) {
-    try {
+      const search = parsed[i].SearchQuery
-      let extracted = extractJSON(genResponse)
+      // const data = await queryScraper(search);
-      repaired = jsonrepair(extracted);
+      // const output = await rankAndDisplayData(data, search);
-    } catch (repairErr: any) {
-      logger.error("Failed to repair JSON from LLM response.");
-      logger.error("Original LLM response:\n%s", genResponse);
-      throw new Error(`JSON repair failed: ${repairErr.message}`);
-    }
 
-    let parsed;
+      // parsed[i].context = output;
-    try {
+      parsed[i].context = "NONE"
-      const json = JSON.parse(repaired);
-
-      if (Array.isArray(json)) {
-        parsed = ProposedTriggerEventArray.parse(json);
-      } else {
-        // try grab first value
-        const firstValue = Object.values(json)[0];
-
-        if (Array.isArray(firstValue)) {
-          parsed = ProposedTriggerEventArray.parse(firstValue);
-        } else {
-          logger.error("No array found in JSON after parsing.");
-          logger.error("Repaired JSON:\n%s", repaired);
-          logger.error("Original LLM response:\n%s", genResponse);
-          throw new Error("No array found in JSON structure");
-        }
-      }
-    } catch (parseErr: any) {
-      logger.error("Failed to parse LLM response to JSON or validate array.");
-      logger.error("Repaired JSON:\n%s", repaired);
-      logger.error("Original LLM response:\n%s", genResponse);
-      throw new Error(`Parsing failed: ${parseErr.message}`);
     }
     
     return { proposedTriggerEvent: parsed, proposedTriggerEventIndex: 0 };
-  } else {
+  }
-    logger.info("Trigger event index %s", state.proposedTriggerEventIndex + 1);
+  else {
+    logger.info("Trigger event index %s", state.proposedTriggerEventIndex+1)
     
-    return { proposedTriggerEvent: state.proposedTriggerEvent, proposedTriggerEventIndex: state.proposedTriggerEventIndex + 1 };
+    return { proposedTriggerEvent: state.proposedTriggerEvent, proposedTriggerEventIndex: state.proposedTriggerEventIndex+1 };
   }
 };

agent/package.json

@@ -17,7 +17,6 @@
     "@langchain/core": "^1.1.17",
     "@langchain/langgraph": "^1.1.2",
     "@langchain/langgraph-sdk": "^1.5.5",
-    "@langchain/ollama": "^1.2.6",
     "@langchain/openai": "^1.2.3",
     "axios": "^1.13.5",
     "compute-cosine-similarity": "^1.1.0",
@@ -25,7 +24,6 @@
     "dotenv": "^17.2.3",
     "exponential-backoff": "^3.1.3",
     "fs": "^0.0.1-security",
-    "jsonrepair": "^3.13.3",
     "langchain": "^1.2.14",
     "selenium-webdriver": "^4.40.0",
     "tldts": "^7.0.23",

agent/prompts/normalization.txt

@@ -16,7 +16,4 @@ Relevent examples are included in preceeding messages, use these as exact inspir
 The claim to normalize is:
 ###TITLE###
 
-Produce no other text other than the condensed claim, surrounded <norm></norm>
+Produce no other text other than the condensed claim.
-
-For example: BREAKING: the sky is green!
-Becomes: <norm>The sky is green</norm>

agent/prompts/relation.txt

@@ -0,0 +1,9 @@
+Could the following real-world event:
+###TECLAIM###
+
+Be a trigger for the following disinformation:
+###TITLE###
+
+Respond with "RELATION", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
+
+Ignore wether the event happened or not, purely consider the likiness of causation

agent/prompts/trigger.txt

@@ -3,10 +3,9 @@ Once the information has been created as below, a dataset can be created to feed
 
 There is a false disinformation claim circulating:
 ###NTITLE###
-Produce up-to 5 specific "trigger events" that happened that could have led to the spread of this disinformation.
+Produce up-to 5 specific events that happened that have led to the spread of this disinformation.
 
 Remember the time frame of the disinformation campaign: ###CDATE###
-Include no information or events that would not have been available at the time. 
 
 Produce no more text other than the json.
 
@@ -14,29 +13,8 @@ Include a concise but specific search query that can be looked up on a search en
 
 Include a url to a source for your trigger event (not a web search, a specific url from a reputuable source). Do not use OAI cite, include url as text in response.
 
-Include the date that the event happened ("March 2022" for exmaple)
+Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url".
-
-Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,Date".
-Return ONLY JSON, no extra text. Wrap it like this:
-<json>
-[
-    {
-        "Event": "Example"
-        ...
-    }
-]
-</json>
 
 Multiple tool invocations should be requested at once, if applicable.
-Use your abilities to look between the lines and produce some insightful analysis, thinking both short and long term.
-
-Events will be reordered as part of processing, each statement must stand alone
 
 The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given.
-Analysis should only be completed for proposed events that would graner >0.7 points
-
-Since URLs change frequently, use tools to retreive up to date informaiton everytime, provided examples or existing knowledge will be wrong or out of date.
-
-Remember to return just json enclosed by <json></json>
-
-Lets go through it step by step

agent/prompts/verify.txt

@@ -0,0 +1,8 @@
+Do the search results cited below
+###TESEARCH###
+Support the idea that the following happened:
+###TECLAIM###
+
+Respond with "CONFIDENCE", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
+
+Dates can be off by a few days, that would still be valid

agent/state.ts

@@ -9,7 +9,6 @@ export const ProposedTriggerEvent = z.object({
   ReasoningWhyRelevant: z.string(),
   SearchQuery: z.string(),
   Url: z.url(),
-  Date: z.string(),
   context: z.string().optional(),
   score: z.number().optional()
 })

agent/tools/ensembleCall.ts

@@ -7,7 +7,7 @@ export async function evaluateWithEnsemble({
   answer: string;
   method: string
 }): Promise<{ validProb: number; invalidProb: number; }> {
-    const res = await axios.post(process.env.RANKING_URL ?? "http://localhost:8000/evaluate", {
+  const res = await axios.post("http://localhost:8000/evaluate", {
     answer,
     method
   }, {timeout: 0});
@@ -18,15 +18,11 @@ export async function evaluateWithEnsemble({
   return {validProb, invalidProb};
 }
 
-// import dotenv from "dotenv";
+// let res = await evaluateWithRoberta({answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
-
-// dotenv.config();
-
-// let res = await evaluateWithEnsemble({method:"flan" ,answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
 // console.log(res)
 
-// res = await evaluateWithEnsemble({method:"roberta" ,answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
+// res = await evaluateWithRoberta({answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
 // console.log(res)
 
-// res = await evaluateWithEnsemble({method:"logreg" ,answer: "The COVID-19 Pandemic"});
+// res = await evaluateWithRoberta({answer: "The COVID-19 Pandemic"});
 // console.log(res)

agent/tools/ragasCall.ts

@@ -0,0 +1,22 @@
+import axios from "axios";
+
+export async function evaluateWithRagas({
+  question,
+  answer,
+  contexts,
+}: {
+  question: string;
+  answer: string;
+  contexts: string[];
+}) {
+  const res = await axios.post("http://localhost:8001/evaluate", {
+    question,
+    answer,
+    contexts,
+  });
+
+  return res.data;
+}
+
+// let res = await evaluateWithRagas({question: "Who was Bill Nye", answer: "Bill Nye was a Scientist", contexts: ["Bill nye was a Scientist"]});
+// console.log(res)

agent/tools/retreiveExamples.ts

@@ -15,8 +15,6 @@ const CACHE_PATH = "../data/csv.cache.json";
 
 const JSONL_PATH = "../data/input.jsonl"
 
-const BM25_MIN_DOCS = 3;
-
 type EmbeddingCache = {
   rawtexts: string[];
   cleantexts: string[];
@@ -289,20 +287,8 @@ async function embedText(text: string): Promise<number[]> {
 }
 
 function buildBM25(texts: string[]) {
-  let paddedTexts = texts;
+  logger.info("Building BM25 index (%s docs)...", texts.length);
 
-  if (texts.length < BM25_MIN_DOCS) {
-    const needed = BM25_MIN_DOCS - texts.length;
-    logger.error(
-      "Corpus too small for BM25 (%s docs, need %s+), padding with %s dummy doc(s)",
-      texts.length,
-      BM25_MIN_DOCS,
-      needed
-    );
-    paddedTexts = [...texts, ...Array(needed).fill("placeholder dummy document")];
-  }
-
-  logger.info("Building BM25 index (%s docs)...", paddedTexts.length);
   const bm25 = bm25Factory();
 
   bm25.defineConfig({
@@ -316,7 +302,7 @@ function buildBM25(texts: string[]) {
     nlp.tokens.removeWords,
   ]);
 
-  paddedTexts.forEach((text, i) => {
+  texts.forEach((text, i) => {
     bm25.addDoc({ text }, i);
   });
 

agent/tools/webpageFetch.ts

@@ -1,95 +1,32 @@
 import { Builder, Browser } from "selenium-webdriver";
 import firefox from "selenium-webdriver/firefox";
-import { backOff } from "exponential-backoff";
-import { logger } from "../utils/logger";
 
-export async function extractWebpageContent(url: string): Promise<string[]> {
+export async function extractWebpageContent(url: string) : Promise<string[]>{
-  try {
-    const response = await backOff(async () => {
-      return await extractWebpageContentWorker(url);
-    }, {
-      numOfAttempts: 10,
-      startingDelay: 500,
-      timeMultiple: 2,
-      jitter: "full",
-      maxDelay: 50000,
-    });
-    return response;
-  } catch (err: any) {
-    logger.error(`Failed out of retry loop for URL "${url}", returning placeholder to pipeline`);
-    return ["API EXCEPTION"];
-  }
-}
-
-async function extractWebpageContentWorker(url: string): Promise<string[]> {
-  let driver;
-  try {
     const options = new firefox.Options();
     options.addArguments("--headless");
-    options.addArguments("--disable-gpu");
-    options.addArguments("--no-sandbox"); // Linux sandbox issues
-    options.addArguments("--disable-dev-shm-usage"); // /dev/shm issues
-    driver = await new Builder()
-      .forBrowser(Browser.FIREFOX)
-      .setFirefoxOptions(options)
-      .build();
-  } catch (err: any) {
-    const desc = `Failed to launch Firefox driver: ${err.message}`;
-    logger.error(desc);
-    throw new Error(desc);
-  }
 
-  try {
+    let driver = await new Builder().forBrowser(Browser.FIREFOX).setFirefoxOptions(options).build()
     try {
-      await driver.get(url);
+        await driver.get(url)
-    } catch (err: any) {
+        await driver.wait(async () => {
-      const desc = `Failed to navigate to URL "${url}": ${err.message}`;
+            return await driver.executeScript(
-      logger.error(desc);
+                "return document.readyState === 'complete'"
-      throw new Error(desc);
+            );
-    }
+        }, 5000);
 
-    try {
+        const readableText = await driver.executeScript(
-      await driver.wait(async () => {
+            "return document.body.innerText;"
-        return await driver.executeScript(
+        ) as string;
-          "return document.readyState === 'complete'"
-        );
-      }, 5000);
-    } catch (err: any) {
-      logger.error(`Page load timed out for "${url}", attempting to read partial content: ${err.message}`);
-      // do not throw, attempt to read
-    }
 
-    let readableText: string;
+        const filteredLines = readableText
-    try {
+            .split(/\r?\n/)
-      readableText = await driver.executeScript(
+            .map(line => line.trim())
-        "return document.body.innerText;"
+            .filter(line => line.split(/\s+/).length > 1); 
-      ) as string;
-    } catch (err: any) {
-      const desc = `Failed to extract page text from "${url}": ${err.message}`;
-      logger.error(desc);
-      throw new Error(desc);
-    }
         
-    const filteredLines = readableText
+        return filteredLines;
-      .split(/\r?\n/)
+    } finally {
-      .map(line => line.trim())
+        await driver.quit()
-      .filter(line => line.split(/\s+/).length > 1);
-
-    if (filteredLines.length === 0) {
-      const desc = `No content extracted from "${url}"`;
-      logger.error(desc);
-      throw new Error(desc);
     }
-
-    return filteredLines;
-  } finally {
-    try {
-      await driver.quit();
-    } catch (err: any) {
-      logger.error(`Failed to quit Firefox driver cleanly: ${err.message}`);
-    }
-  }
 }
 
-// console.log(await extractWebpageContent("https://www.bbc.co.uk/news/live/c74wd01egvyt"))
+//console.log(await extractWebpageContent("https://www.bbc.co.uk/news/live/c74wd01egvyt"))
-// console.log(await extractWebpageContent("https://badcertificate.int.jeynes.uk/"))

supporting/RAGAS_Service/ensemble_service.py

@@ -92,7 +92,7 @@ LABEL_TO_INT = {v: k for k, v in INT_TO_LABEL.items()}
 flan_tokenizer = AutoTokenizer.from_pretrained(FLAN_PATH)
 flan_model = AutoModelForSeq2SeqLM.from_pretrained(FLAN_PATH)
 
-device = torch.device("cpu")
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 flan_model.to(device)
 flan_model.eval()
 

supporting/RAGAS_Service/requirements.txt

@@ -9,7 +9,6 @@ datasets
 # ROBERTA
 scikit-learn
 transformers[torch]
-sentence_transformers
 
 # Utils
 numpy

supporting/Wrapper/run.ts

@@ -17,7 +17,7 @@ const AGENT_NAME = process.env.AGENT ?? "agent";
  */
 const MODE = process.env.MODE ?? "claim";
 
-const MAX_CONCURRENCY = 1;
+const MAX_CONCURRENCY = 5;
 
 const client = new Client({ apiUrl: API_URL });
 
@@ -118,7 +118,7 @@ async function processRecord(record: any): Promise<ResultRecord> {
       input: buildAgentInput(record),
       streamMode: "values",
       config: {
-        recursion_limit: 100
+        recursion_limit: 50
       }
     });