# -*- coding: utf-8 -*-
"""Axium vs Orange vs Hermes — three harnesses, one model, one grader.

Refuses to print a comparison it cannot stand behind:
  * every row must post-date the 20:04 UTC grader rewrite on the one axis it changed (V4)
  * agents with unequal n are flagged, not silently averaged
"""
import collections
import io
import json
import os

FIX = '2026-08-06T20:04:00+00:00'
SRC = {
    'axium': r'C:\xampp\htdocs\axium\python\versus\logs\axium.jsonl',
    'orange': r'C:\xampp\htdocs\axium\python\versus\logs\orange.jsonl',
    'hermes': r'C:\agent-eval-thirdparty\eval\logs\hermes.jsonl',
}
AGENTS = ['axium', 'orange', 'hermes']
AXES = {'V1': 'repair', 'V2': 'restraint', 'V3': 'continuity',
        'V4': 'blast radius', 'V5': 'economy'}
IDS = ['V1', 'V2', 'V3', 'V4', 'V5']


def load(a):
    p = SRC[a]
    return ([json.loads(l) for l in io.open(p, encoding='utf-8') if l.strip()]
            if os.path.exists(p) else [])


def detail(r):
    out = []
    for it in (r.get('change_detail') or []):
        if isinstance(it, (list, tuple)) and len(it) == 2:
            out.append((it[0], bool(it[1])))
        elif isinstance(it, dict):
            out.append((it.get('name'), bool(it.get('ok'))))
    return out


def score(r):
    d = detail(r)
    return 100.0 * sum(1 for _, ok in d if ok) / len(d) if d else None


DATA = {a: load(a) for a in AGENTS}
BY = {a: collections.defaultdict(list) for a in AGENTS}
for a in AGENTS:
    for r in DATA[a]:
        BY[a][r['id']].append(r)

print('=' * 86)
print('VALIDITY')
print('=' * 86)
blocked = []
for a in AGENTS:
    bad = [r for r in DATA[a] if r['id'] == 'V4' and r['stamp'] < FIX]
    ns = {k: len(BY[a][k]) for k in IDS}
    complete = all(ns.get(k, 0) >= 3 for k in IDS)
    print('  %-7s %2d rows  per-axis n=%s  %s%s'
          % (a, len(DATA[a]), [ns.get(k, 0) for k in IDS],
             'OK' if not bad else 'V4 PRE-FIX ROWS: %d' % len(bad),
             '' if complete else '   INCOMPLETE (<3 reps on some axis)'))
    if bad or not complete:
        blocked.append(a)
print()

usable = [a for a in AGENTS if a not in blocked and DATA[a]]

print('=' * 86)
print('CHANGE SCORE BY AXIS   (regress was 100%% for every agent in every session)')
print('=' * 86)
hdr = '  %-4s %-14s' % ('id', 'axis') + ''.join('%12s' % a.upper() for a in usable) + '   best'
print(hdr)
print('  ' + '-' * (len(hdr) - 2))
tot = {a: 0.0 for a in usable}
for sid in IDS:
    cells, vals = '', {}
    for a in usable:
        rs = BY[a].get(sid) or []
        if rs:
            m = sum(score(r) for r in rs) / len(rs)
            vals[a] = m
            tot[a] += m
            cells += '%11.1f%%' % m
        else:
            cells += '%12s' % '-'
    best = ''
    if vals:
        top = max(vals.values())
        winners = [a for a, v in vals.items() if abs(v - top) < 0.05]
        best = 'tie' if len(winners) > 1 else winners[0]
    print('  %-4s %-14s%s   %s' % (sid, AXES[sid], cells, best))
print('  ' + '-' * (len(hdr) - 2))
mean = {a: tot[a] / len(IDS) for a in usable}
print('  %-19s' % 'MEAN' + ''.join('%11.1f%%' % mean[a] for a in usable)
      + '   %s' % (max(mean, key=mean.get) if mean else ''))
print()

print('=' * 86)
print('COST AND EFFORT PER SESSION')
print('=' * 86)
print('  %-8s %8s %10s %12s %11s %10s %10s'
      % ('agent', 'sessions', 'total $', '$/session', 'calls/sess', 'min/sess', 'vs cheapest'))
print('  ' + '-' * 78)
per = {}
for a in usable:
    rs = DATA[a]
    c = sum((r.get('metrics') or {}).get('cost_usd', 0) for r in rs)
    k = sum((r.get('metrics') or {}).get('llm_calls', 0) for r in rs)
    w = sum(r.get('wall_s') or 0 for r in rs)
    per[a] = (len(rs), c, c / len(rs), k / len(rs), w / len(rs) / 60.0)
cheap = min(per[a][2] for a in usable) if usable else 1
for a in usable:
    n, c, cs, kc, wm = per[a]
    print('  %-8s %8d %10.4f %12.5f %11.1f %10.1f %9.2fx' % (a, n, c, cs, kc, wm, cs / cheap))
print()

print('=' * 86)
print('CHECKS THAT FAILED IN EVERY RUN  (n>=3; the only reproducible signal)')
print('=' * 86)
for a in usable:
    hits = []
    for sid, rs in BY[a].items():
        cnt, tt = collections.Counter(), collections.Counter()
        for r in rs:
            for nm, ok in detail(r):
                tt[nm] += 1
                if not ok:
                    cnt[nm] += 1
        for nm, n in cnt.items():
            if n == tt[nm] and n >= 3:
                hits.append((sid, nm, n))
    print('  %s:' % a)
    for sid, nm, n in sorted(hits) or [(None, None, None)]:
        print('    (none)' if sid is None else '    %-4s %-50s %d/%d' % (sid, nm, n, n))
    print()

if blocked:
    print('!! NOT COMPARABLE YET: %s — rerun before quoting the mean.' % ', '.join(blocked))