Getting hits on the block list IMMEDIATLEY. Log to file, might be important later

This commit is contained in:
William Jeynes
2026-02-22 15:42:27 +00:00
parent 2f33338007
commit 4d92f14527
3 changed files with 8 additions and 0 deletions
+2
View File
@@ -1,5 +1,6 @@
import axios from "axios";
import { checkDisinfo } from "./checkDisinfo";
import { writeToJSONL } from "../utils/writeToJSONL";
export async function queryScraper(query: string): Promise<string[]> {
const instance = process.env.SCRAPER_INSTANCE;
@@ -46,6 +47,7 @@ export async function queryScraper(query: string): Promise<string[]> {
const lines: string[] = context.map((item: any) => {
if (checkDisinfo(item.url)) {
writeToJSONL("blocked.jsonl", {url: item.url, query: query})
return "";
}