From a2cb93b44e56b4e8a6a4990939564532703f2cf4 Mon Sep 17 00:00:00 2001 From: William Jeynes Date: Wed, 18 Feb 2026 15:03:13 +0000 Subject: [PATCH] Start refining scorer. Filter data passed to trigger event agent --- agent/nodes/triggerEventSetup.ts | 6 +- agent/prompts/trigger.txt | 10 ++-- agent/state.ts | 1 - agent/tools/retreiveExamples.ts | 6 +- supporting/scorer/display.py | 100 +++++++++++++++++++------------ supporting/scorer/rules.txt | 4 ++ 6 files changed, 77 insertions(+), 50 deletions(-) create mode 100644 supporting/scorer/rules.txt diff --git a/agent/nodes/triggerEventSetup.ts b/agent/nodes/triggerEventSetup.ts index f2b5b75..4cf36f8 100644 --- a/agent/nodes/triggerEventSetup.ts +++ b/agent/nodes/triggerEventSetup.ts @@ -1,7 +1,7 @@ import { GraphNode } from "@langchain/langgraph"; import { MessagesState } from "../state"; -import { AIMessage, BaseMessage, HumanMessage } from "@langchain/core/messages"; -import { rankExampleTriggerEvents, rankNormalizedClaims } from "../tools/retreiveExamples"; +import { AIMessage, BaseMessage } from "@langchain/core/messages"; +import { rankExampleTriggerEvents } from "../tools/retreiveExamples"; export const triggerEventSetup: GraphNode = async (state) => { let nc = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp @@ -10,7 +10,7 @@ export const triggerEventSetup: GraphNode = async (state) let similarityResults = await rankExampleTriggerEvents(state.disinformationTitle) let messages : BaseMessage[] = similarityResults.map((item) => { - return new AIMessage(`Event: ${item.rawtext}, Claims and given scores: ${item.cleantext}`) + return new AIMessage(`- Event: ${item.rawtext} \n\n - Claims and given scores: ${item.cleantext}`) }) return { messages: messages, disinformationTitle: state.disinformationTitle, normalizedClaim: nc }; diff --git a/agent/prompts/trigger.txt b/agent/prompts/trigger.txt index 7076aef..52e77df 100644 --- a/agent/prompts/trigger.txt +++ b/agent/prompts/trigger.txt @@ -2,7 +2,7 @@ You are an agent in a pipeline to analyse disinformation. Once the information has been created as below, a dataset can be created to feed a model for prediction, which will improve pre-bunking efforts. There is a false disinformation claim circulating: -("###NTITLE### +###NTITLE### Produce up-to 5 specific "trigger events" that happened that could have led to the spread of this disinformation. Remember the time frame of the disinformation campaign: {{CAMPAIGN_DATE}} @@ -14,16 +14,14 @@ Include a concise but specific search query that can be looked up on a search en Include a url to a source for your trigger event (not a web search, a specific url from a reputuable source). Do not use OAI cite, include url as text in response. -If you are referencing another disinformation campaign, provide the specific narrative used, not just sentiment, and ensure it is sufficiently different from the claim we are analysing. - -Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,IsItselfDisinformation". - +Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url". Multiple tool invocations should be requested at once, if applicable. Use your abilities to look between the lines and produce some insightful analysis, thinking both short and long term. Events will be reordered as part of processing, each statement must stand alone -The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given, to help understanding the intended quality of analysis +The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given. +Analysis should only be completed for proposed events that would graner >0.7 points Lets go through it step by step \ No newline at end of file diff --git a/agent/state.ts b/agent/state.ts index 9a69b7f..b4388fc 100644 --- a/agent/state.ts +++ b/agent/state.ts @@ -9,7 +9,6 @@ export const ProposedTriggerEvent = z.object({ ReasoningWhyRelevant: z.string(), SearchQuery: z.string(), Url: z.url(), - IsItselfDisinformation: z.boolean(), context: z.string().optional(), score: z.number().optional() }) diff --git a/agent/tools/retreiveExamples.ts b/agent/tools/retreiveExamples.ts index c595c68..d26d823 100644 --- a/agent/tools/retreiveExamples.ts +++ b/agent/tools/retreiveExamples.ts @@ -257,7 +257,11 @@ async function ensureExampleClaimJsonlLoaded(): Promise { jsonlRawtexts.push(text); - jsonlCleantexts.push(row.output[0].content); + const parsed_content = row.output[0].content_parsed; + + const filtered_content = parsed_content.filter(itm => itm.human_score > 0.5 && itm.score > 0.5) + + jsonlCleantexts.push(JSON.stringify(filtered_content)); jsonlEmbeddings.push(embedding); } diff --git a/supporting/scorer/display.py b/supporting/scorer/display.py index 2a1150c..f91ed83 100644 --- a/supporting/scorer/display.py +++ b/supporting/scorer/display.py @@ -24,6 +24,7 @@ def load_data(file_path): o["content_parsed"] = json.loads(o["content"]) except json.JSONDecodeError: o["content_parsed"] = [] + print("parse error") data.append(entry) return data @@ -68,59 +69,80 @@ if view == "All Claims": st.markdown("---") # -------------------------- -# Single Claim Random View +# Single Claim Random View (Ranking Based) # -------------------------- elif view == "Single Claim Random": - # Rebuild the list of unscored claims only when needed + + # Select an entry that still has unscored items if st.session_state.current_claim is None: - single_claims = [] + unscored_entries = [] + for entry in st.session_state.data: - url = entry.get("documentUrl") - text = entry.get("text") + unscored = [] + for o in entry.get("output", []): for c in o.get("content_parsed", []): - if "human_score" not in c or c.get("human_score") is None: - single_claims.append({ - "documentUrl": url, - "text": text, - "event": c.get("event"), - "reasoningWhyRelevant": c.get("reasoningWhyRelevant"), - "raw_obj": c # reference to original object - }) - if single_claims: - st.session_state.current_claim = random.choice(single_claims) + if c.get("human_score") is None: + unscored.append(c) + + if unscored: + unscored_entries.append({ + "entry": entry, + "claims": unscored + }) + + if unscored_entries: + st.session_state.current_claim = random.choice(unscored_entries) else: st.session_state.current_claim = None - claim = st.session_state.current_claim + bundle = st.session_state.current_claim - if claim is None: - st.info("No claims available without a human score.") + if bundle is None: + st.info("No entries remaining without human scores.") else: - st.subheader(f"{claim['text']}") - st.markdown(f"**Event:** {claim['event']}") - st.markdown(f"**Reasoning:** {claim['reasoningWhyRelevant']}") + entry = bundle["entry"] + claims = bundle["claims"] - # Input for new human score - new_score = st.number_input( - "Provide a score (0 to 1)", - min_value=0.0, - max_value=1.0, - value=0.5, - step=0.01, - format="%.2f" - ) + st.subheader(entry.get("text")) - if st.button("Submit Score"): - # Update the original object - claim["raw_obj"]["human_score"] = new_score + st.write("Rank events (1 = best / most relevant)") + + rankings = [] + + # Collect rankings + for i, c in enumerate(claims): + st.markdown(f"### Event {i+1}") + st.markdown(f"**Event:** {c.get('event')}") + st.markdown(f"**Reasoning:** {c.get('reasoningWhyRelevant')}") + + rank = st.number_input( + f"Rank for event {i+1}", + min_value=1, + max_value=len(claims), + key=f"rank_{i}" + ) + + rankings.append((c, rank)) + + if st.button("Submit Ranking"): + + # Sort by rank (ascending = best) + rankings.sort(key=lambda x: x[1]) + + n = len(rankings) + + # Convert ranking -> normalized score + for idx, (claim_obj, _) in enumerate(rankings): + if n == 1: + score = 1.0 + else: + score = 1 - (idx / (n - 1)) + + claim_obj["human_score"] = round(score, 3) - # Save immediately save_data(DATA_FILE, st.session_state.data) - st.success("Score saved!") + st.success("Ranking converted to scores and saved!") - # Clear current claim so a new one will be selected next run st.session_state.current_claim = None - - # Rerun app to show a new claim - st.rerun() + st.rerun() \ No newline at end of file diff --git a/supporting/scorer/rules.txt b/supporting/scorer/rules.txt new file mode 100644 index 0000000..df599a3 --- /dev/null +++ b/supporting/scorer/rules.txt @@ -0,0 +1,4 @@ +1. Proposed trigger events rewording the initial prompt should penalised +2. Proposed trigger events should be specific enough to create resonable analysis +3. Proposed trigger events should be time-specific. A trigger events that happened after the event is invalid +4. Proposed trigger events should not describe a story of propogation \ No newline at end of file