Getting hits on the block list IMMEDIATLEY. Log to file, might be important later

README.md

@@ -11,6 +11,7 @@ Final Dissertation Submission Repository
 ```
 ├── run.sh                          # Bash script to run project elements from one place
 ├── data/                           # Holder from project data
+|   ├── blocked.jsonl               # Web search results blocked by the Iffy list
 |   ├── claims.json                 # Retreived claims from dbkf fetcher
 |   ├── dev-eng.csv
 |   ├── train-eng.csv               # Normalized disinformation claims in CSV format from CLAN

agent/tools/webSearch.ts

@@ -1,5 +1,6 @@
 import axios from "axios";
 import { checkDisinfo } from "./checkDisinfo";
+import { writeToJSONL } from "../utils/writeToJSONL";
 
 export async function queryScraper(query: string): Promise<string[]> {
     const instance = process.env.SCRAPER_INSTANCE;
@@ -46,6 +47,7 @@ export async function queryScraper(query: string): Promise<string[]> {
 
     const lines: string[] = context.map((item: any) => {
         if (checkDisinfo(item.url)) {
+            writeToJSONL("blocked.jsonl", {url: item.url, query: query})
             return "";
         }
 

agent/utils/writeToJSONL.ts

@@ -0,0 +1,5 @@
+import fs from "fs";
+
+export function writeToJSONL(path: string, line: any) {
+    fs.appendFileSync(`../data/${path}`, JSON.stringify(line) + "\n", "utf-8");
+}