Add removing of duplicates from pipeline. Add to sort step. Move score logic to robertaMetrics node.

2026-03-13 14:51:14 +00:00
parent d5c6cb444d
commit 0a7bb114d2
6 changed files with 105 additions and 7 deletions
@@ -82,7 +82,7 @@ def render():
                    extra_lower = (event.get("extra_info", "") or "").strip().lower()
                    # print(extra_lower)
                    if score is not None:
-                        if "duplicate" in extra_lower:
+                        if score == -1 or "duplicate" in extra_lower:
                            dup_counter += 1
                        elif score > THRESH and extra_lower == "perfect":
                            confidence_counter["Correct-PERFECT"] += 1