Add a chain of thought to the main prompt

agent/prompts/trigger.txt

@@ -3,9 +3,10 @@ Once the information has been created as below, a dataset can be created to feed
 
 There is a false disinformation claim circulating:
 ###NTITLE###
-Produce up-to 5 specific events that happened that have led to the spread of this disinformation.
+Produce up-to 5 specific "trigger events" that happened that could have led to the spread of this disinformation.
 
 Remember the time frame of the disinformation campaign: ###CDATE###
+Include no information or events that would not have been available at the time. 
 
 Produce no more text other than the json.
 
@@ -13,8 +14,20 @@ Include a concise but specific search query that can be looked up on a search en
 
 Include a url to a source for your trigger event (not a web search, a specific url from a reputuable source). Do not use OAI cite, include url as text in response.
 
-Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url".
+Include the date that the event happened ("March 2022" for exmaple)
+
+Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,Date".
 
 Multiple tool invocations should be requested at once, if applicable.
+Use your abilities to look between the lines and produce some insightful analysis, thinking both short and long term.
 
-The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given.
+Events will be reordered as part of processing, each statement must stand alone
+
+The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given.
+Analysis should only be completed for proposed events that would graner >0.7 points
+
+First, consider a range of directions in which the proposed disinformation could have been influenced by.
+Then, research these directions in turn, using the tools at hand.
+Finally, refine your proposed "trigger event" until it is specific, quantifiable and backed up by evidence.
+
+Lets go through it step by step

agent/state.ts

@@ -9,6 +9,7 @@ export const ProposedTriggerEvent = z.object({
   ReasoningWhyRelevant: z.string(),
   SearchQuery: z.string(),
   Url: z.url(),
+  Date: z.string(),
   context: z.string().optional(),
   score: z.number().optional()
 })

agent/tools/retreiveExamples.ts

@@ -15,6 +15,8 @@ const CACHE_PATH = "../data/csv.cache.json";
 
 const JSONL_PATH = "../data/input.jsonl"
 
+const BM25_MIN_DOCS = 3;
+
 type EmbeddingCache = {
   rawtexts: string[];
   cleantexts: string[];
@@ -287,8 +289,20 @@ async function embedText(text: string): Promise<number[]> {
 }
 
 function buildBM25(texts: string[]) {
-  logger.info("Building BM25 index (%s docs)...", texts.length);
+  let paddedTexts = texts;
 
+  if (texts.length < BM25_MIN_DOCS) {
+    const needed = BM25_MIN_DOCS - texts.length;
+    logger.error(
+      "Corpus too small for BM25 (%s docs, need %s+), padding with %s dummy doc(s)",
+      texts.length,
+      BM25_MIN_DOCS,
+      needed
+    );
+    paddedTexts = [...texts, ...Array(needed).fill("placeholder dummy document")];
+  }
+
+  logger.info("Building BM25 index (%s docs)...", paddedTexts.length);
   const bm25 = bm25Factory();
 
   bm25.defineConfig({
@@ -302,7 +316,7 @@ function buildBM25(texts: string[]) {
     nlp.tokens.removeWords,
   ]);
 
-  texts.forEach((text, i) => {
+  paddedTexts.forEach((text, i) => {
     bm25.addDoc({ text }, i);
   });
 

agent/tools/webpageFetch.ts

@@ -1,32 +1,92 @@
 import { Builder, Browser } from "selenium-webdriver";
 import firefox from "selenium-webdriver/firefox";
+import { backOff } from "exponential-backoff";
+import { logger } from "../utils/logger";
 
-export async function extractWebpageContent(url: string) : Promise<string[]>{
+export async function extractWebpageContent(url: string): Promise<string[]> {
-    const options = new firefox.Options();
+  try {
-    options.addArguments("--headless");
+    const response = await backOff(async () => {
-
+      return await extractWebpageContentWorker(url);
-    let driver = await new Builder().forBrowser(Browser.FIREFOX).setFirefoxOptions(options).build()
+    }, {
-    try {
+      numOfAttempts: 10,
-        await driver.get(url)
+      startingDelay: 500,
-        await driver.wait(async () => {
+      timeMultiple: 2,
-            return await driver.executeScript(
+      jitter: "full",
-                "return document.readyState === 'complete'"
+      maxDelay: 50000,
-            );
+    });
-        }, 5000);
+    return response;
-
+  } catch (err: any) {
-        const readableText = await driver.executeScript(
+    logger.error(`Failed out of retry loop for URL "${url}", returning placeholder to pipeline`);
-            "return document.body.innerText;"
+    return ["API EXCEPTION"];
-        ) as string;
+  }
-
-        const filteredLines = readableText
-            .split(/\r?\n/)
-            .map(line => line.trim())
-            .filter(line => line.split(/\s+/).length > 1); 
-        
-        return filteredLines;
-    } finally {
-        await driver.quit()
-    }
 }
 
-//console.log(await extractWebpageContent("https://www.bbc.co.uk/news/live/c74wd01egvyt"))
+async function extractWebpageContentWorker(url: string): Promise<string[]> {
+  let driver;
+  try {
+    const options = new firefox.Options();
+    options.addArguments("--headless");
+    driver = await new Builder()
+      .forBrowser(Browser.FIREFOX)
+      .setFirefoxOptions(options)
+      .build();
+  } catch (err: any) {
+    const desc = `Failed to launch Firefox driver: ${err.message}`;
+    logger.error(desc);
+    throw new Error(desc);
+  }
+
+  try {
+    try {
+      await driver.get(url);
+    } catch (err: any) {
+      const desc = `Failed to navigate to URL "${url}": ${err.message}`;
+      logger.error(desc);
+      throw new Error(desc);
+    }
+
+    try {
+      await driver.wait(async () => {
+        return await driver.executeScript(
+          "return document.readyState === 'complete'"
+        );
+      }, 5000);
+    } catch (err: any) {
+      logger.error(`Page load timed out for "${url}", attempting to read partial content: ${err.message}`);
+      // do not throw, attempt to read
+    }
+
+    let readableText: string;
+    try {
+      readableText = await driver.executeScript(
+        "return document.body.innerText;"
+      ) as string;
+    } catch (err: any) {
+      const desc = `Failed to extract page text from "${url}": ${err.message}`;
+      logger.error(desc);
+      throw new Error(desc);
+    }
+
+    const filteredLines = readableText
+      .split(/\r?\n/)
+      .map(line => line.trim())
+      .filter(line => line.split(/\s+/).length > 1);
+
+    if (filteredLines.length === 0) {
+      const desc = `No content extracted from "${url}"`;
+      logger.error(desc);
+      throw new Error(desc);
+    }
+
+    return filteredLines;
+  } finally {
+    try {
+      await driver.quit();
+    } catch (err: any) {
+      logger.error(`Failed to quit Firefox driver cleanly: ${err.message}`);
+    }
+  }
+}
+
+// console.log(await extractWebpageContent("https://www.bbc.co.uk/news/live/c74wd01egvyt"))
+// console.log(await extractWebpageContent("https://badcertificate.int.jeynes.uk/"))

supporting/Wrapper/run.ts

@@ -118,7 +118,7 @@ async function processRecord(record: any): Promise<ResultRecord> {
       input: buildAgentInput(record),
       streamMode: "values",
       config: {
-        recursion_limit: 50
+        recursion_limit: 100
       }
     });