diff --git a/agent/agent.ts b/agent/agent.ts index 0539f88..5be1b57 100644 --- a/agent/agent.ts +++ b/agent/agent.ts @@ -5,19 +5,17 @@ import { createToolConditional } from "./conditionals/tool_end"; import { normalizationSetup } from "./nodes/normalizationSetup"; import { triggerEventToolsByName } from "./tools/triggerEventTools" import { verificationSetup } from "./nodes/verificationSetup"; -import { ragasMetrics } from "./nodes/ragasMetrics"; import { produceRanking } from "./nodes/produceRanking"; import { createModelNode } from "./nodes/model"; import { loopEndConditional } from "./conditionals/loop_end"; import { sort } from "./nodes/sort"; import { triggerEventSetup } from "./nodes/triggerEventSetup"; +import { robertaMetrics } from "./nodes/robertaMetrics"; const triggerEventToolNode = createToolNode(triggerEventToolsByName); const normalisationModel = createModelNode([], "normalization.txt"); const triggerEventModel = createModelNode(triggerEventToolsByName, "trigger.txt"); -const verificationModel = createModelNode([], "verify.txt"); -const relationModel = createModelNode([], "relation.txt"); const triggerEventToolConditional = createToolConditional("triggerEventToolNode", verificationSetup.name); @@ -32,9 +30,7 @@ const agent = new StateGraph(MessagesState) .addNode("triggerEventModel", triggerEventModel) .addNode(verificationSetup.name, verificationSetup) - .addNode("verificationModel", verificationModel) - .addNode(ragasMetrics.name, ragasMetrics) - .addNode("relationModel", relationModel) + .addNode(robertaMetrics.name, robertaMetrics) .addNode(produceRanking.name, produceRanking) .addNode(sort.name, sort) @@ -49,13 +45,9 @@ const agent = new StateGraph(MessagesState) .addConditionalEdges("triggerEventModel", triggerEventToolConditional, ["triggerEventToolNode", verificationSetup.name]) .addEdge("triggerEventToolNode", "triggerEventModel") - .addEdge(verificationSetup.name, "verificationModel") - .addEdge(verificationSetup.name, ragasMetrics.name) - .addEdge(verificationSetup.name, "relationModel") - - .addEdge(ragasMetrics.name, produceRanking.name) - .addEdge("verificationModel", produceRanking.name) - .addEdge("relationModel", produceRanking.name) + .addEdge(verificationSetup.name, robertaMetrics.name) + + .addEdge(robertaMetrics.name, produceRanking.name) // @ts-expect-error .addConditionalEdges(produceRanking.name, loopEndConditional, [verificationSetup.name, sort.name]) diff --git a/supporting/dbkf/fetch.py b/supporting/dbkf/fetch.py index f5e25f8..99dbfb7 100644 --- a/supporting/dbkf/fetch.py +++ b/supporting/dbkf/fetch.py @@ -16,18 +16,18 @@ BASE_URL = "https://dbkf.ontotext.com/rest-api/search/documents" # "documentTypes": "http://schema.org/Claim", DEFAULT_PARAMS = [ - ("concept", "http://weverify.eu/resource/Concept/Q84263196"), + ("concept", "http://weverify.eu/resource/Concept/Q212"), ("from", "2000-01-01"), ("to", "2026-02-19"), ("lang", "en"), ("limit", 5000), ("page", 1), ("orderBy", "date"), - ("organization", "http://weverify.eu/resource/Organization/72b4f61c7cb49873004bea24f0a8f8f9"), # PolitifactFB - ("organization", "http://weverify.eu/resource/Organization/552abae8eb4e003e69a3351eb0eae372") # LeadStories + ("organization", "http://weverify.eu/resource/Organization/3727f7b2aa90ec0716693e5464b28d18"), # StopFake + ("organization", "http://weverify.eu/resource/Organization/c71953fa6cf24ac4178f751c77862070"), # CheckYourFact ] -NUM_RANDOM_CLAIMS = 20 +NUM_RANDOM_CLAIMS = 40 INPUT_FILE = "../../data/input.jsonl" OUTPUT_FILE = "../../data/claims.json" diff --git a/supporting/scorer/data_utils.py b/supporting/scorer/data_utils.py index 4b3f757..014cd77 100644 --- a/supporting/scorer/data_utils.py +++ b/supporting/scorer/data_utils.py @@ -11,69 +11,14 @@ def load_data(file_path): continue entry = json.loads(line) - outputs = entry.get("output", []) - - if isinstance(outputs, dict): - outputs = [outputs] - - for o in outputs: - content = o.get("content") - if content: - try: - o["content_parsed"] = json.loads(content) - except json.JSONDecodeError: - o["content_parsed"] = [] - - entry["output"] = outputs data.append(entry) return data def save_data_clean(file_path, data): - merged = {} - - for entry in data: - events = [] - for o in entry.get("output", []): - if "content_parsed" in o: - events.extend(o["content_parsed"]) - - doc_url = entry.get("documentUrl") - if not doc_url: - continue - - if doc_url not in merged: - new_entry = entry.copy() - new_entry["events"] = events - new_entry.pop("output", None) - new_entry.pop("status", None) - new_entry["run_id"] - merged[doc_url] = new_entry - - - else: - merged[doc_url]["events"].extend(events) - - for entry in merged.values(): - entry["events"].sort( - key=lambda e: e.get("human_score", 0), - reverse=True - ) - - with open(file_path, "w", encoding="utf-8") as f: - for entry in merged.values(): - f.write(json.dumps(entry, ensure_ascii=False) + "\n") - - -def save_data(file_path, data): with open(file_path, "w", encoding="utf-8") as f: for entry in data: - for o in entry.get("output", []): - if "content_parsed" in o: - o["content"] = json.dumps( - o["content_parsed"], - ensure_ascii=False - ) f.write(json.dumps(entry, ensure_ascii=False) + "\n") + diff --git a/supporting/scorer/views/all_claims.py b/supporting/scorer/views/all_claims.py index bb45699..51ca92d 100644 --- a/supporting/scorer/views/all_claims.py +++ b/supporting/scorer/views/all_claims.py @@ -8,11 +8,9 @@ def render(): for entry in st.session_state.data: st.subheader(entry.get("text")) - for o in entry.get("output", []): - for c in o.get("content_parsed", []): - st.markdown(f"**Event:** {c.get('event')}") - st.markdown(f"**Reasoning:** {c.get('reasoningWhyRelevant')}") - st.markdown(f"**Score:** {c.get('score')}") - st.markdown(f"**Human Score:** {c.get('human_score')}") - st.markdown(f"**Extra Info:** {c.get('extra_info', '')}") - st.markdown("---") \ No newline at end of file + for c in entry.get("events", []): + st.markdown(f"**Event:** {c.get('Event')}") + st.markdown(f"**Reasoning:** {c.get('ReasoningWhyRelevant')}") + st.markdown(f"**Score:** {c.get('score')}") + st.markdown(f"**Extra Info:** {c.get('extra_info', '')}") + st.markdown("---") \ No newline at end of file diff --git a/supporting/scorer/views/label.py b/supporting/scorer/views/label.py index dcce216..61c28da 100644 --- a/supporting/scorer/views/label.py +++ b/supporting/scorer/views/label.py @@ -1,7 +1,7 @@ import random import streamlit as st from config import INPUT_FILE -from data_utils import save_data +from data_utils import save_data_clean def page_title() -> str: @@ -15,10 +15,10 @@ def render(): for entry in st.session_state.data: claims = [] - for o in entry.get("output", []): - for c in o.get("content_parsed", []): - if not c.get("ranked"): - claims.append(c) + + for c in entry.get("events", []): + if not c.get("ranked"): + claims.append(c) if claims: unannotated.append({"entry": entry, "claims": claims}) @@ -44,8 +44,8 @@ def render(): with st.container(border=True): - st.markdown(f"**Event:** {c.get('event')}") - st.markdown(f"**Reasoning:** {c.get('reasoningWhyRelevant')}") + st.markdown(f"**Event:** {c.get('Event')}") + st.markdown(f"**Reasoning:** {c.get('ReasoningWhyRelevant')}") cols = st.columns(7) temp = "" @@ -69,7 +69,7 @@ def render(): c["ranked"] = True if st.button("Save Annotation"): - save_data(INPUT_FILE, st.session_state.data) + save_data_clean(INPUT_FILE, st.session_state.data) st.session_state.current_claim = None print("Annotation saved") st.rerun() \ No newline at end of file diff --git a/supporting/scorer/views/rank.py b/supporting/scorer/views/rank.py deleted file mode 100644 index 3c76e93..0000000 --- a/supporting/scorer/views/rank.py +++ /dev/null @@ -1,116 +0,0 @@ -import streamlit as st -import copy -import random -from streamlit_sortables import sort_items -from config import INPUT_FILE, OUTPUT_FILE -from data_utils import save_data, save_data_clean - - -def page_title() -> str: - return "Rank" - -def render(): - st.header("Rank Events") - candidates = [] - - for entry in st.session_state.data: - perfect = [] - - for o in entry.get("output", []): - for c in o.get("content_parsed", []): - if "PERFECT" in c.get("extra_info", "") and not c.get("rank_position"): - perfect.append(c) - - if perfect: - candidates.append({"entry": entry, "claims": perfect}) - - if not candidates: - st.info("No events available.") - st.stop() - - if "current_bundle" not in st.session_state: - st.session_state.current_bundle = random.choice(candidates) - - bundle = st.session_state.current_bundle - entry = bundle["entry"] - claims = bundle["claims"] - - st.subheader(entry.get("text")) - - # init - if "perfect_order" not in st.session_state: - st.session_state.perfect_order = list(range(len(claims))) - - order = st.session_state.perfect_order - - # labels shown in sortable UI - labels = [ - f"{i+1}. {claims[idx].get('event')}" - for i, idx in enumerate(order) - ] - - st.markdown("### Drag to reorder:") - - # ------------------------- - # Drag & drop UI - # ------------------------- - new_labels = sort_items(labels) - - # Convert reordered labels back → indices - if new_labels != labels: - new_order = [] - for lbl in new_labels: - original_pos = labels.index(lbl) - new_order.append(order[original_pos]) - - st.session_state.perfect_order = new_order - order = new_order - - st.markdown("---") - for rank, idx in enumerate(order): - c = claims[idx] - st.markdown(f"**Rank {rank+1}: {c.get('event')}**") - st.markdown(c.get("reasoningWhyRelevant")) - st.markdown("---") - - if st.button("Submit Ranking"): - - n = len(order) - - for rank_position, idx in enumerate(order): - claim_obj = claims[idx] - - # explicit stored rank - claim_obj["rank_position"] = rank_position + 1 - - claim_obj["human_score"] = 1 - - # Auto-scoring - for entry in st.session_state.data: - for o in entry.get("output", []): - for c in o.get("content_parsed", []): - - if c.get("human_score") is not None: - continue - - extra = c.get("extra_info", "") - - if "DUPLICATE" in extra: - c["human_score"] = 0 - elif extra: - c["human_score"] = round( - c.get("score", 0) * 0.5, 3 - ) - - save_data(INPUT_FILE, st.session_state.data) - save_data_clean( - OUTPUT_FILE, - copy.deepcopy(st.session_state.data) - ) - - # reset state for next example - del st.session_state.current_bundle - del st.session_state.perfect_order - - print("Ranking saved!") - st.rerun() \ No newline at end of file diff --git a/supporting/scorer/views/stats.py b/supporting/scorer/views/stats.py index c30c39c..cbc9826 100644 --- a/supporting/scorer/views/stats.py +++ b/supporting/scorer/views/stats.py @@ -18,16 +18,14 @@ def render(): # ---- collect stats ---- for entry in st.session_state.data: - for o in entry.get("output", []): - for c in o.get("content_parsed", []): + for c in entry.get("events", []): + # ---- extra_info word counts ---- + extra = c.get("extra_info", "") + score = c.get("score", None) - # ---- extra_info word counts ---- - extra = c.get("extra_info", "") - score = c.get("score", None) - - if extra: - words = extra.strip().split() - word_counter.update(words) + if extra: + words = extra.strip().split() + word_counter.update(words) # -------------------------- # Extra Info Word Counts