Allow multiple source CSV files for normalisation. Implement real model node. Add normalizarion prompt. Implement normalization setup. Start on RAG retreival functions

2026-02-09 16:32:40 +00:00
parent 8eaa7bfbff
commit 02eac0f553
9 changed files with 311 additions and 56 deletions
@@ -0,0 +1,29 @@
+import { Builder, Browser } from "selenium-webdriver";
+import firefox from "selenium-webdriver/firefox";
+
+async function extractWebpageContent(url: string) : Promise<string>{
+    const options = new firefox.Options();
+    options.addArguments("--headless");
+
+    let driver = await new Builder().forBrowser(Browser.FIREFOX).setFirefoxOptions(options).build()
+    try {
+        await driver.get(url)
+        await driver.wait(async () => {
+            return await driver.executeScript(
+                "return document.readyState === 'complete'"
+            );
+        }, 5000);
+
+        const readableText = await driver.executeScript(
+            "return document.body.innerText;"
+        ) as string;
+
+        return readableText
+    } finally {
+        await driver.quit()
+    }
+}
+
+//TODO: Extract, rank snippets
+
+//console.log(await extractWebpageContent("https://www.bbc.co.uk/news/live/c74wd01egvyt"))