compare.py (1443B)
1 #!/usr/bin/env python3 2 """Compare direct vs shared scoring outputs for argmax agreement and drift.""" 3 import json 4 import sys 5 from pathlib import Path 6 7 BASE = Path.cwd() # validate.sh runs from the repo root 8 9 def load(path): 10 with open(BASE / path) as handle: 11 return {row["id"]: row for row in map(json.loads, handle)} 12 13 def argmax(row): 14 best = max(range(len(row["probabilities"])), key=lambda i: row["probabilities"][i]) 15 return row["option_ids"][best] 16 17 def main(): 18 direct_path = sys.argv[1] if len(sys.argv) > 1 else "results-direct-on-shared.jsonl" 19 shared_path = sys.argv[2] if len(sys.argv) > 2 else "results-shared.jsonl" 20 direct, shared = load(direct_path), load(shared_path) 21 if set(direct) != set(shared): 22 print(f"ID mismatch: only-direct={sorted(set(direct) - set(shared))} " 23 f"only-shared={sorted(set(shared) - set(direct))}") 24 flips = 0 25 for rid in sorted(set(direct) & set(shared)): 26 d, s = direct[rid], shared[rid] 27 match = argmax(d) == argmax(s) 28 flips += not match 29 print(f"{rid}: argmax {'MATCH' if match else 'FLIP '} " 30 f"({argmax(d)} vs {argmax(s)})") 31 print(f" direct: {[round(p, 4) for p in d['probabilities']]}") 32 print(f" shared: {[round(p, 4) for p in s['probabilities']]}") 33 print(f"\n{flips} argmax flip(s) across {len(set(direct) & set(shared))} rows") 34 35 if __name__ == "__main__": 36 main()