Update agent to support new verification style. Update frontend to support new file format and remove redundant logic from old experiments.

agent/agent.ts

@@ -5,19 +5,17 @@ import { createToolConditional } from "./conditionals/tool_end";
 import { normalizationSetup } from "./nodes/normalizationSetup";
 import { triggerEventToolsByName } from "./tools/triggerEventTools"
 import { verificationSetup } from "./nodes/verificationSetup";
-import { ragasMetrics } from "./nodes/ragasMetrics";
 import { produceRanking } from "./nodes/produceRanking";
 import { createModelNode } from "./nodes/model";
 import { loopEndConditional } from "./conditionals/loop_end";
 import { sort } from "./nodes/sort";
 import { triggerEventSetup } from "./nodes/triggerEventSetup";
+import { robertaMetrics } from "./nodes/robertaMetrics";
 
 const triggerEventToolNode = createToolNode(triggerEventToolsByName);
 
 const normalisationModel = createModelNode([], "normalization.txt");
 const triggerEventModel = createModelNode(triggerEventToolsByName, "trigger.txt");
-const verificationModel = createModelNode([], "verify.txt");
-const relationModel = createModelNode([], "relation.txt");
 
 const triggerEventToolConditional = createToolConditional("triggerEventToolNode", verificationSetup.name);
 
@@ -32,9 +30,7 @@ const agent = new StateGraph(MessagesState)
   .addNode("triggerEventModel", triggerEventModel)
 
   .addNode(verificationSetup.name, verificationSetup)
-  .addNode("verificationModel", verificationModel)
+  .addNode(robertaMetrics.name, robertaMetrics)
-  .addNode(ragasMetrics.name, ragasMetrics)
-  .addNode("relationModel", relationModel)
   
   .addNode(produceRanking.name, produceRanking)
   .addNode(sort.name, sort)
@@ -49,13 +45,9 @@ const agent = new StateGraph(MessagesState)
   .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", verificationSetup.name])
   .addEdge("triggerEventToolNode", "triggerEventModel")
   
-  .addEdge(verificationSetup.name, "verificationModel")
+  .addEdge(verificationSetup.name, robertaMetrics.name)
-  .addEdge(verificationSetup.name, ragasMetrics.name)
-  .addEdge(verificationSetup.name, "relationModel")
   
-  .addEdge(ragasMetrics.name, produceRanking.name)
+  .addEdge(robertaMetrics.name, produceRanking.name)
-  .addEdge("verificationModel", produceRanking.name)
-  .addEdge("relationModel", produceRanking.name)
 
   // @ts-expect-error
   .addConditionalEdges(produceRanking.name, loopEndConditional, [verificationSetup.name, sort.name])

supporting/dbkf/fetch.py

@@ -16,18 +16,18 @@ BASE_URL = "https://dbkf.ontotext.com/rest-api/search/documents"
 
 # "documentTypes": "http://schema.org/Claim",
 DEFAULT_PARAMS = [
-    ("concept", "http://weverify.eu/resource/Concept/Q84263196"),
+    ("concept", "http://weverify.eu/resource/Concept/Q212"),
     ("from", "2000-01-01"),
     ("to", "2026-02-19"),
     ("lang", "en"),
     ("limit", 5000),
     ("page", 1),
     ("orderBy", "date"),
-    ("organization", "http://weverify.eu/resource/Organization/72b4f61c7cb49873004bea24f0a8f8f9"), # PolitifactFB
+    ("organization", "http://weverify.eu/resource/Organization/3727f7b2aa90ec0716693e5464b28d18"), # StopFake
-    ("organization", "http://weverify.eu/resource/Organization/552abae8eb4e003e69a3351eb0eae372") # LeadStories
+    ("organization", "http://weverify.eu/resource/Organization/c71953fa6cf24ac4178f751c77862070"), # CheckYourFact
 ]
 
-NUM_RANDOM_CLAIMS = 20
+NUM_RANDOM_CLAIMS = 40
 
 INPUT_FILE = "../../data/input.jsonl"
 OUTPUT_FILE = "../../data/claims.json"

supporting/scorer/data_utils.py

@@ -11,69 +11,14 @@ def load_data(file_path):
                     continue
 
                 entry = json.loads(line)
-                outputs = entry.get("output", [])
-
-                if isinstance(outputs, dict):
-                    outputs = [outputs]
-
-                for o in outputs:
-                    content = o.get("content")
-                    if content:
-                        try:
-                            o["content_parsed"] = json.loads(content)
-                        except json.JSONDecodeError:
-                            o["content_parsed"] = []
-
-                entry["output"] = outputs
                 data.append(entry)
 
     return data
 
 
 def save_data_clean(file_path, data):
-    merged = {}
-
-    for entry in data:
-        events = []
-        for o in entry.get("output", []):
-            if "content_parsed" in o:
-                events.extend(o["content_parsed"])
-
-        doc_url = entry.get("documentUrl")
-        if not doc_url:
-            continue
-
-        if doc_url not in merged:
-            new_entry = entry.copy()
-            new_entry["events"] = events
-            new_entry.pop("output", None)
-            new_entry.pop("status", None)
-            new_entry["run_id"]
-            merged[doc_url] = new_entry
-
-            
-        else:
-            merged[doc_url]["events"].extend(events)
-
-    for entry in merged.values():
-        entry["events"].sort(
-            key=lambda e: e.get("human_score", 0),
-            reverse=True
-        )
-
-    with open(file_path, "w", encoding="utf-8") as f:
-        for entry in merged.values():
-            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
-
-
-def save_data(file_path, data):
     with open(file_path, "w", encoding="utf-8") as f:
         for entry in data:
-            for o in entry.get("output", []):
-                if "content_parsed" in o:
-                    o["content"] = json.dumps(
-                        o["content_parsed"],
-                        ensure_ascii=False
-                    )
             f.write(json.dumps(entry, ensure_ascii=False) + "\n")
 
+

supporting/scorer/views/all_claims.py

@@ -8,11 +8,9 @@ def render():
     for entry in st.session_state.data:
         st.subheader(entry.get("text"))
 
-        for o in entry.get("output", []):
+        for c in entry.get("events", []):
-            for c in o.get("content_parsed", []):
+            st.markdown(f"**Event:** {c.get('Event')}")
-                st.markdown(f"**Event:** {c.get('event')}")
+            st.markdown(f"**Reasoning:** {c.get('ReasoningWhyRelevant')}")
-                st.markdown(f"**Reasoning:** {c.get('reasoningWhyRelevant')}")
+            st.markdown(f"**Score:** {c.get('score')}")
-                st.markdown(f"**Score:** {c.get('score')}")
+            st.markdown(f"**Extra Info:** {c.get('extra_info', '')}")
-                st.markdown(f"**Human Score:** {c.get('human_score')}")
+            st.markdown("---")
-                st.markdown(f"**Extra Info:** {c.get('extra_info', '')}")
-                st.markdown("---")

supporting/scorer/views/label.py

@@ -1,7 +1,7 @@
 import random
 import streamlit as st
 from config import INPUT_FILE
-from data_utils import save_data
+from data_utils import save_data_clean
 
 
 def page_title() -> str:
@@ -15,10 +15,10 @@ def render():
         for entry in st.session_state.data:
             claims = []
 
-            for o in entry.get("output", []):
+
-                for c in o.get("content_parsed", []):
+            for c in entry.get("events", []):
-                    if not c.get("ranked"):
+                if not c.get("ranked"):
-                        claims.append(c)
+                    claims.append(c)
 
             if claims:
                 unannotated.append({"entry": entry, "claims": claims})
@@ -44,8 +44,8 @@ def render():
 
             with st.container(border=True):
 
-                st.markdown(f"**Event:** {c.get('event')}")
+                st.markdown(f"**Event:** {c.get('Event')}")
-                st.markdown(f"**Reasoning:** {c.get('reasoningWhyRelevant')}")
+                st.markdown(f"**Reasoning:** {c.get('ReasoningWhyRelevant')}")
 
                 cols = st.columns(7)
                 temp = ""
@@ -69,7 +69,7 @@ def render():
                 c["ranked"] = True
 
         if st.button("Save Annotation"):
-            save_data(INPUT_FILE, st.session_state.data)
+            save_data_clean(INPUT_FILE, st.session_state.data)
             st.session_state.current_claim = None
             print("Annotation saved")
             st.rerun()

supporting/scorer/views/rank.py

@@ -1,116 +0,0 @@
-import streamlit as st
-import copy
-import random
-from streamlit_sortables import sort_items
-from config import INPUT_FILE, OUTPUT_FILE
-from data_utils import save_data, save_data_clean
-
-
-def page_title() -> str:
-    return "Rank"
-
-def render():
-    st.header("Rank Events")
-    candidates = []
-
-    for entry in st.session_state.data:
-        perfect = []
-
-        for o in entry.get("output", []):
-            for c in o.get("content_parsed", []):
-                if "PERFECT" in c.get("extra_info", "") and not c.get("rank_position"):
-                    perfect.append(c)
-
-        if perfect:
-            candidates.append({"entry": entry, "claims": perfect})
-
-    if not candidates:
-        st.info("No events available.")
-        st.stop()
-
-    if "current_bundle" not in st.session_state:
-        st.session_state.current_bundle = random.choice(candidates)
-
-    bundle = st.session_state.current_bundle
-    entry = bundle["entry"]
-    claims = bundle["claims"]
-
-    st.subheader(entry.get("text"))
-
-    # init
-    if "perfect_order" not in st.session_state:
-        st.session_state.perfect_order = list(range(len(claims)))
-
-    order = st.session_state.perfect_order
-
-    # labels shown in sortable UI
-    labels = [
-        f"{i+1}. {claims[idx].get('event')}"
-        for i, idx in enumerate(order)
-    ]
-
-    st.markdown("### Drag to reorder:")
-
-    # -------------------------
-    # Drag & drop UI
-    # -------------------------
-    new_labels = sort_items(labels)
-
-    # Convert reordered labels back → indices
-    if new_labels != labels:
-        new_order = []
-        for lbl in new_labels:
-            original_pos = labels.index(lbl)
-            new_order.append(order[original_pos])
-
-        st.session_state.perfect_order = new_order
-        order = new_order
-
-    st.markdown("---")
-    for rank, idx in enumerate(order):
-        c = claims[idx]
-        st.markdown(f"**Rank {rank+1}: {c.get('event')}**")
-        st.markdown(c.get("reasoningWhyRelevant"))
-        st.markdown("---")
-
-    if st.button("Submit Ranking"):
-
-        n = len(order)
-
-        for rank_position, idx in enumerate(order):
-            claim_obj = claims[idx]
-
-            # explicit stored rank
-            claim_obj["rank_position"] = rank_position + 1
-
-            claim_obj["human_score"] = 1
-
-        # Auto-scoring
-        for entry in st.session_state.data:
-            for o in entry.get("output", []):
-                for c in o.get("content_parsed", []):
-
-                    if c.get("human_score") is not None:
-                        continue
-
-                    extra = c.get("extra_info", "")
-
-                    if "DUPLICATE" in extra:
-                        c["human_score"] = 0
-                    elif extra:
-                        c["human_score"] = round(
-                            c.get("score", 0) * 0.5, 3
-                        )
-
-        save_data(INPUT_FILE, st.session_state.data)
-        save_data_clean(
-            OUTPUT_FILE,
-            copy.deepcopy(st.session_state.data)
-        )
-
-        # reset state for next example
-        del st.session_state.current_bundle
-        del st.session_state.perfect_order
-
-        print("Ranking saved!")
-        st.rerun()

supporting/scorer/views/stats.py

@@ -18,16 +18,14 @@ def render():
 
     # ---- collect stats ----
     for entry in st.session_state.data:
-        for o in entry.get("output", []):
+        for c in entry.get("events", []):
-            for c in o.get("content_parsed", []):
+            # ---- extra_info word counts ----
+            extra = c.get("extra_info", "")
+            score = c.get("score", None)
 
-                # ---- extra_info word counts ----
+            if extra:
-                extra = c.get("extra_info", "")
+                words = extra.strip().split()
-                score = c.get("score", None)
+                word_counter.update(words)
-
-                if extra:
-                    words = extra.strip().split()
-                    word_counter.update(words)
 
     # --------------------------
     # Extra Info Word Counts