diff --git a/agent/nodes/robertaMetrics.ts b/agent/nodes/robertaMetrics.ts index 2763a39..6bfd7d6 100644 --- a/agent/nodes/robertaMetrics.ts +++ b/agent/nodes/robertaMetrics.ts @@ -8,10 +8,9 @@ export const robertaMetrics: GraphNode = async (state) => const result = await evaluateWithRoberta({answer}) - let score = 0; - if (result.validProb > (result.invalidProb+0.4)) { - score = 0.7 + ((result.validProb - (result.invalidProb+0.4))*0.3); - } + + const score = result.validProb - result.invalidProb; + return { messages: [ new AIMessage("ROBERTA:" + score)] diff --git a/supporting/RAGAS_Service/generate_adversarial.py b/supporting/RAGAS_Service/generate_adversarial.py index 6e6240c..fdbb311 100644 --- a/supporting/RAGAS_Service/generate_adversarial.py +++ b/supporting/RAGAS_Service/generate_adversarial.py @@ -24,13 +24,17 @@ def make_request(line): try: data = json.loads(line) prompt = ( - "Provide a story item for the spread of a disinformation claim" - "that is related to the topic: " + "Provide a non specific piece of background, tallking point or other minformaiton that allowed the a disinformaition to spread; to aid in analysis and debunking" + "The topic in question is: " + data.get("text", "") - + " Include just the event no other text." - + " A good example would be 'No immediate U.S. government confirmation and near‑simultaneous fact‑checks/debunks appeared (fact‑checks published June 26, 2024).' and 'Recycled/old footage of aircraft being shot down previously viral and repeatedly misattributed to the Russia–Ukraine war (e.g., 2011 Libya footage reused in 2022)'" - + " If you cannot answer just return an empty string" - + " Be concise, make no mistakes" + + " Include just the example no other text." + + " A good example would be" + + "'Existing high-profile reporting and public discussion throughout 2022–2023 about foreign fighters and mercenary recruitment (including Russian recruitment and Wagner Group activity).'" + + "Since it focusses on non-instantiated or proven discussion points" + + " and " + + "'2016 – Continued EU–Russia business dialogues and investments (documented by policy institutes and trade analyses) showing ongoing economic links despite political tensions'" + + "Since it does not name a specific dialogue or investement" + + " Be concise, make no mistakes, use similar style and wording to provided examples" ) if not prompt: diff --git a/supporting/scorer/views/stats.py b/supporting/scorer/views/stats.py index 5c623d2..4da21cc 100644 --- a/supporting/scorer/views/stats.py +++ b/supporting/scorer/views/stats.py @@ -5,7 +5,8 @@ import streamlit as st import pandas as pd import matplotlib.pyplot as plt -THRESH = 0.7 +# THRESH = 0.4 +THRESH = 0.6 def page_title() -> str: return "Statistics" @@ -121,6 +122,8 @@ def render(): goodkept = confidence_counter["Correct-PERFECT"] + confidence_counter["Correct-FINE"] allkept = confidence_counter["Correct-PERFECT"] + confidence_counter["Correct-FINE"] + confidence_counter["Over-confident"] + if (allkept == 0): + allkept = -1 corr_percent = (correct / total) * 100 kept_percent = (goodkept / allkept) * 100