Getting hits on the block list IMMEDIATLEY. Log to file, might be important later

agent/tools/webSearch.ts

@@ -1,5 +1,6 @@
 import axios from "axios";
 import { checkDisinfo } from "./checkDisinfo";
+import { writeToJSONL } from "../utils/writeToJSONL";
 
 export async function queryScraper(query: string): Promise<string[]> {
     const instance = process.env.SCRAPER_INSTANCE;
@@ -46,6 +47,7 @@ export async function queryScraper(query: string): Promise<string[]> {
 
     const lines: string[] = context.map((item: any) => {
         if (checkDisinfo(item.url)) {
+            writeToJSONL("blocked.jsonl", {url: item.url, query: query})
             return "";
         }
 

agent/utils/writeToJSONL.ts

@@ -0,0 +1,5 @@
+import fs from "fs";
+
+export function writeToJSONL(path: string, line: any) {
+    fs.appendFileSync(`../data/${path}`, JSON.stringify(line) + "\n", "utf-8");
+}