jill/LLMsForDisinformationAnalysis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update agent to support new verification style. Update frontend to support new file format and remove redundant logic from old experiments.

Browse Source
This commit is contained in:
William Jeynes
2026-03-16 17:16:58 +00:00
parent 0a7bb114d2
commit c89c7054fe
7 changed files with 31 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
supporting/dbkf/fetch.py
+4 -4
View File
@@ -16,18 +16,18 @@ BASE_URL = "https://dbkf.ontotext.com/rest-api/search/documents"
# "documentTypes": "http://schema.org/Claim",
DEFAULT_PARAMS = [
("concept", "http://weverify.eu/resource/Concept/Q84263196"),
("concept", "http://weverify.eu/resource/Concept/Q212"),
("from", "2000-01-01"),
("to", "2026-02-19"),
("lang", "en"),
("limit", 5000),
("page", 1),
("orderBy", "date"),
("organization", "http://weverify.eu/resource/Organization/72b4f61c7cb49873004bea24f0a8f8f9"), # PolitifactFB
("organization", "http://weverify.eu/resource/Organization/552abae8eb4e003e69a3351eb0eae372") # LeadStories
("organization", "http://weverify.eu/resource/Organization/3727f7b2aa90ec0716693e5464b28d18"), # StopFake
("organization", "http://weverify.eu/resource/Organization/c71953fa6cf24ac4178f751c77862070"), # CheckYourFact
]
NUM_RANDOM_CLAIMS = 20
NUM_RANDOM_CLAIMS = 40
INPUT_FILE = "../../data/input.jsonl"
OUTPUT_FILE = "../../data/claims.json"
supporting/scorer/data_utils.py
+1 -56
View File
@@ -11,69 +11,14 @@ def load_data(file_path):
continue
entry = json.loads(line)
outputs = entry.get("output", [])
if isinstance(outputs, dict):
outputs = [outputs]
for o in outputs:
content = o.get("content")
if content:
try:
o["content_parsed"] = json.loads(content)
except json.JSONDecodeError:
o["content_parsed"] = []
entry["output"] = outputs
data.append(entry)
return data
def save_data_clean(file_path, data):
merged = {}
for entry in data:
events = []
for o in entry.get("output", []):
if "content_parsed" in o:
events.extend(o["content_parsed"])
doc_url = entry.get("documentUrl")
if not doc_url:
continue
if doc_url not in merged:
new_entry = entry.copy()
new_entry["events"] = events
new_entry.pop("output", None)
new_entry.pop("status", None)
new_entry["run_id"]
merged[doc_url] = new_entry
else:
merged[doc_url]["events"].extend(events)
for entry in merged.values():
entry["events"].sort(
key=lambda e: e.get("human_score", 0),
reverse=True
)
with open(file_path, "w", encoding="utf-8") as f:
for entry in merged.values():
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
def save_data(file_path, data):
with open(file_path, "w", encoding="utf-8") as f:
for entry in data:
for o in entry.get("output", []):
if "content_parsed" in o:
o["content"] = json.dumps(
o["content_parsed"],
ensure_ascii=False
)
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
supporting/scorer/views/all_claims.py
+6 -8
View File
@@ -8,11 +8,9 @@ def render():
for entry in st.session_state.data:
st.subheader(entry.get("text"))
for o in entry.get("output", []):
for c in o.get("content_parsed", []):
st.markdown(f"**Event:** {c.get('event')}")
st.markdown(f"**Reasoning:** {c.get('reasoningWhyRelevant')}")
st.markdown(f"**Score:** {c.get('score')}")
st.markdown(f"**Human Score:** {c.get('human_score')}")
st.markdown(f"**Extra Info:** {c.get('extra_info', '')}")
st.markdown("---")
for c in entry.get("events", []):
st.markdown(f"**Event:** {c.get('Event')}")
st.markdown(f"**Reasoning:** {c.get('ReasoningWhyRelevant')}")
st.markdown(f"**Score:** {c.get('score')}")
st.markdown(f"**Extra Info:** {c.get('extra_info', '')}")
st.markdown("---")
supporting/scorer/views/label.py
+8 -8
View File
@@ -1,7 +1,7 @@
import random
import streamlit as st
from config import INPUT_FILE
from data_utils import save_data
from data_utils import save_data_clean
def page_title() -> str:
@@ -15,10 +15,10 @@ def render():
for entry in st.session_state.data:
claims = []
for o in entry.get("output", []):
for c in o.get("content_parsed", []):
if not c.get("ranked"):
claims.append(c)
for c in entry.get("events", []):
if not c.get("ranked"):
claims.append(c)
if claims:
unannotated.append({"entry": entry, "claims": claims})
@@ -44,8 +44,8 @@ def render():
with st.container(border=True):
st.markdown(f"**Event:** {c.get('event')}")
st.markdown(f"**Reasoning:** {c.get('reasoningWhyRelevant')}")
st.markdown(f"**Event:** {c.get('Event')}")
st.markdown(f"**Reasoning:** {c.get('ReasoningWhyRelevant')}")
cols = st.columns(7)
temp = ""
@@ -69,7 +69,7 @@ def render():
c["ranked"] = True
if st.button("Save Annotation"):
save_data(INPUT_FILE, st.session_state.data)
save_data_clean(INPUT_FILE, st.session_state.data)
st.session_state.current_claim = None
print("Annotation saved")
st.rerun()
supporting/scorer/views/rank.py
-116
View File
@@ -1,116 +0,0 @@
import streamlit as st
import copy
import random
from streamlit_sortables import sort_items
from config import INPUT_FILE, OUTPUT_FILE
from data_utils import save_data, save_data_clean
def page_title() -> str:
return "Rank"
def render():
st.header("Rank Events")
candidates = []
for entry in st.session_state.data:
perfect = []
for o in entry.get("output", []):
for c in o.get("content_parsed", []):
if "PERFECT" in c.get("extra_info", "") and not c.get("rank_position"):
perfect.append(c)
if perfect:
candidates.append({"entry": entry, "claims": perfect})
if not candidates:
st.info("No events available.")
st.stop()
if "current_bundle" not in st.session_state:
st.session_state.current_bundle = random.choice(candidates)
bundle = st.session_state.current_bundle
entry = bundle["entry"]
claims = bundle["claims"]
st.subheader(entry.get("text"))
# init
if "perfect_order" not in st.session_state:
st.session_state.perfect_order = list(range(len(claims)))
order = st.session_state.perfect_order
# labels shown in sortable UI
labels = [
f"{i+1}. {claims[idx].get('event')}"
for i, idx in enumerate(order)
]
st.markdown("### Drag to reorder:")
# -------------------------
# Drag & drop UI
# -------------------------
new_labels = sort_items(labels)
# Convert reordered labels back → indices
if new_labels != labels:
new_order = []
for lbl in new_labels:
original_pos = labels.index(lbl)
new_order.append(order[original_pos])
st.session_state.perfect_order = new_order
order = new_order
st.markdown("---")
for rank, idx in enumerate(order):
c = claims[idx]
st.markdown(f"**Rank {rank+1}: {c.get('event')}**")
st.markdown(c.get("reasoningWhyRelevant"))
st.markdown("---")
if st.button("Submit Ranking"):
n = len(order)
for rank_position, idx in enumerate(order):
claim_obj = claims[idx]
# explicit stored rank
claim_obj["rank_position"] = rank_position + 1
claim_obj["human_score"] = 1
# Auto-scoring
for entry in st.session_state.data:
for o in entry.get("output", []):
for c in o.get("content_parsed", []):
if c.get("human_score") is not None:
continue
extra = c.get("extra_info", "")
if "DUPLICATE" in extra:
c["human_score"] = 0
elif extra:
c["human_score"] = round(
c.get("score", 0) * 0.5, 3
)
save_data(INPUT_FILE, st.session_state.data)
save_data_clean(
OUTPUT_FILE,
copy.deepcopy(st.session_state.data)
)
# reset state for next example
del st.session_state.current_bundle
del st.session_state.perfect_order
print("Ranking saved!")
st.rerun()
supporting/scorer/views/stats.py
+7 -9
View File
@@ -18,16 +18,14 @@ def render():
# ---- collect stats ----
for entry in st.session_state.data:
for o in entry.get("output", []):
for c in o.get("content_parsed", []):
for c in entry.get("events", []):
# ---- extra_info word counts ----
extra = c.get("extra_info", "")
score = c.get("score", None)
# ---- extra_info word counts ----
extra = c.get("extra_info", "")
score = c.get("score", None)
if extra:
words = extra.strip().split()
word_counter.update(words)
if extra:
words = extra.strip().split()
word_counter.update(words)
# --------------------------
# Extra Info Word Counts