diff --git a/README.md b/README.md index 97d1b68..f9f8f3c 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Final Dissertation Submission Repository ``` ├── run.sh # Bash script to run project elements from one place ├── data/ # Holder from project data +| ├── blocked.jsonl # Web search results blocked by the Iffy list | ├── claims.json # Retreived claims from dbkf fetcher | ├── dev-eng.csv | ├── train-eng.csv # Normalized disinformation claims in CSV format from CLAN diff --git a/agent/tools/webSearch.ts b/agent/tools/webSearch.ts index 2b47ef0..54aeda0 100644 --- a/agent/tools/webSearch.ts +++ b/agent/tools/webSearch.ts @@ -1,5 +1,6 @@ import axios from "axios"; import { checkDisinfo } from "./checkDisinfo"; +import { writeToJSONL } from "../utils/writeToJSONL"; export async function queryScraper(query: string): Promise { const instance = process.env.SCRAPER_INSTANCE; @@ -46,6 +47,7 @@ export async function queryScraper(query: string): Promise { const lines: string[] = context.map((item: any) => { if (checkDisinfo(item.url)) { + writeToJSONL("blocked.jsonl", {url: item.url, query: query}) return ""; } diff --git a/agent/utils/writeToJSONL.ts b/agent/utils/writeToJSONL.ts new file mode 100644 index 0000000..7dbd54b --- /dev/null +++ b/agent/utils/writeToJSONL.ts @@ -0,0 +1,5 @@ +import fs from "fs"; + +export function writeToJSONL(path: string, line: any) { + fs.appendFileSync(`../data/${path}`, JSON.stringify(line) + "\n", "utf-8"); +} \ No newline at end of file