# -*- coding: utf-8 -*- """Read the session logs and print the paper's tables. No third-party library. python analyse.py everything python analyse.py --suite V --tier T0,T1,T2 Four things, in this order, because the order is the argument: 1. The floor, per scenario, from the controls. Every score below is read against it. 2. The matrix: harness by rung, mean score, raw and floor-adjusted. 3. The decomposition: how much of the spread is the rung and how much is the harness, with the interaction reported rather than assumed away. 4. What it cost and what broke, per cell. A cell that did not run is printed as absent, never as zero, and never dropped quietly from a mean. """ import argparse import glob import io import json import os HERE = os.path.dirname(os.path.abspath(__file__)) LOGS = os.path.join(HERE, 'logs') HARNESS_ORDER = ['null', 'prose', 'axium', 'axium-guard', 'orange', 'winagent', 'hermes', 'dsh', 'openclaw'] PARTY = {'null': 'control', 'prose': 'control', 'axium': 'author', 'axium-guard': 'author', 'orange': 'author', 'winagent': 'author', 'hermes': 'third', 'dsh': 'third', 'openclaw': 'third'} def load(suite='V'): """Every logged session, with the unusable ones separated rather than dropped. A session whose every turn errored measures nothing: its score is whatever the scenario gives an agent that did not run, which is the floor. Averaging those in would report a crashed harness as a cautious one. """ rows = [] for path in sorted(glob.glob(os.path.join(LOGS, '*__%s.jsonl' % suite))): with io.open(path, encoding='utf-8') as f: for line in f: line = line.strip() if line: rows.append(json.loads(line)) return rows def mean(xs): xs = [x for x in xs if x is not None] return sum(xs) / len(xs) if xs else None def fmt(x, nd=3): return ' .' if x is None else ('%.*f' % (nd, x)).rjust(6) def floor_table(rows): ctrl = [r for r in rows if PARTY.get(r['harness']) == 'control'] ids = sorted({r['id'] for r in rows}) print('\nTHE FLOOR, per scenario, from an agent that does nothing') print('%-6s %-14s %s' % ('id', 'axis', ' null prose')) out = {} for sid in ids: n = mean([r['change'] for r in ctrl if r['id'] == sid and r['harness'] == 'null']) p = mean([r['change'] for r in ctrl if r['id'] == sid and r['harness'] == 'prose']) axis = next((r['axis'] for r in rows if r['id'] == sid), '') print('%-6s %-14s %s %s' % (sid, axis[:14], fmt(n), fmt(p))) out[sid] = n return out def adjusted(score, floor): if score is None or floor is None or floor >= 1.0: return None return (score - floor) / (1.0 - floor) def matrix(rows, floors, tiers): print('\nHARNESS BY RUNG, mean of the five scenarios') print('raw score above, floor-adjusted below, sessions in brackets') head = ' ' * 12 + ''.join(t.rjust(9) for t in tiers) print(head) for h in HARNESS_ORDER: hr = [r for r in rows if r['harness'] == h] if not hr: continue raw_cells, adj_cells, counts = [], [], [] for t in tiers: cell = [r for r in hr if r['tier'] == t] counts.append(len(cell)) raw_cells.append(mean([r['change'] for r in cell])) adj_cells.append(mean([adjusted(r['change'], floors.get(r['id'])) for r in cell])) print('%-11s %s' % (h, ''.join((fmt(c) + ' ')[:9] for c in raw_cells))) print('%-11s %s' % ('', ''.join((fmt(c) + ' ')[:9] for c in adj_cells))) print('%-11s %s' % ('', ''.join((' [%d]' % c).rjust(9) for c in counts))) def per_axis(rows, floors, tiers): ids = sorted({r['id'] for r in rows}) print('\nPER AXIS, floor-adjusted, harness by rung') for sid in ids: print(' %s %s' % (sid, next((r['axis'] for r in rows if r['id'] == sid), ''))) for h in HARNESS_ORDER: cells = [] for t in tiers: sub = [r for r in rows if r['harness'] == h and r['tier'] == t and r['id'] == sid] cells.append(mean([adjusted(r['change'], floors.get(sid)) for r in sub])) if all(c is None for c in cells): continue print(' %-9s %s' % (h, ''.join((fmt(c) + ' ')[:9] for c in cells))) def decompose(rows, floors, tiers): """How much of the spread is the rung, how much the harness, how much neither. A two-way decomposition on cell means, unweighted, which is legitimate only while every cell carries the same number of sessions. The counts are printed beside the matrix for exactly that reason: if they differ, this table is read as descriptive rather than as an F test, and it is never reported as one here in any case. """ all_ids = sorted({r['id'] for r in rows}) candidates = [h for h in HARNESS_ORDER if PARTY.get(h) != 'control' and any(r['harness'] == h for r in rows)] # A cell counts only when it holds every scenario in the suite, and a harness # counts only when every rung's cell does. An unbalanced grid makes the three # shares below add to more than the whole, and a cell missing its hardest # scenarios reads as a better cell rather than a shorter one. def full(h, t): ids = {r['id'] for r in rows if r['harness'] == h and r['tier'] == t} return set(all_ids) <= ids hs = [h for h in candidates if all(full(h, t) for t in tiers)] dropped = [h for h in candidates if h not in hs] cells = {} for h in hs: for t in tiers: sub = [r for r in rows if r['harness'] == h and r['tier'] == t] v = mean([adjusted(r['change'], floors.get(r['id'])) for r in sub]) if v is not None: cells[(h, t)] = v if len(cells) < 4: print('\n' + 'DECOMPOSITION: not enough complete cells yet (%d).' % len(cells)) if dropped: print(' incomplete, and therefore not decomposed: %s' % ', '.join(dropped)) return have_t = list(tiers) grand = mean(list(cells.values())) h_means = {h: mean([cells[(h, t)] for t in have_t if (h, t) in cells]) for h in hs} t_means = {t: mean([cells[(h, t)] for h in hs if (h, t) in cells]) for t in have_t} ss_total = sum((v - grand) ** 2 for v in cells.values()) ss_h = sum((h_means[h] - grand) ** 2 for (h, t) in cells) ss_t = sum((t_means[t] - grand) ** 2 for (h, t) in cells) ss_i = sum((cells[(h, t)] - h_means[h] - t_means[t] + grand) ** 2 for (h, t) in cells) print('\n' + 'DECOMPOSITION of the floor-adjusted cell means') print(' balanced grid: %d harness(es) x %d rung(s), grand mean %.3f' % (len(hs), len(have_t), grand)) if dropped: print(' excluded for an incomplete cell: %s' % ', '.join(dropped)) if ss_total <= 0: print(' no spread to decompose') return for label, ss in (('rung (the model)', ss_t), ('harness', ss_h), ('interaction', ss_i)): print(' %-18s %6.1f%% of the spread' % (label, 100 * ss / ss_total)) print(' spread by rung : %s' % ', '.join( '%s %.3f' % (t, t_means[t]) for t in have_t)) print(' spread by harness: %s' % ', '.join( '%s %.3f' % (h, h_means[h]) for h in hs)) def economy(rows, tiers): print('\nWHAT IT COST AND WHAT BROKE') print('%-10s %-7s %8s %9s %9s %8s %7s %6s' % ( 'harness', 'rung', 'usd/sess', 'in', 'out', 'wall_s', 'tools', 'err')) for h in HARNESS_ORDER: for t in tiers: cell = [r for r in rows if r['harness'] == h and r['tier'] == t] if not cell: continue m = [r['metrics'] for r in cell] priced = any(r.get('priced') for r in cell) usd = mean([r.get('cost_usd_recomputed', r['metrics']['cost_usd']) for r in cell]) print('%-10s %-7s %8s %9s %9s %8s %7s %6d' % ( h, t, ('%.5f' % usd) if priced else 'unpriced', '%.0f' % mean([x['input_tokens'] for x in m]), '%.0f' % mean([x['output_tokens'] for x in m]), '%.0f' % mean([x['wall_s'] for x in m]), '%.1f' % mean([x['tool_calls'] for x in m]), sum(len(x['errors']) for x in m))) def regress_note(rows, tiers): broke = [r for r in rows if r['regress'] < 1.0] if not broke: return print('\nDAMAGE: sessions that left the acceptance suite red') for r in sorted(broke, key=lambda r: (r['harness'], r['tier'], r['id'])): print(' %-9s %-6s %-3s rep%s regress %.2f %s' % ( r['harness'], r['tier'], r['id'], r['rep'], r['regress'], (r['regress_misses'] or [''])[0][:60])) def w_table(rows, tiers): """Suite W by completion, which is the measure with a true zero. The row score is kept beside it because the rows carry the restraint half of each scenario, but a mean of binary rows on this suite starts at 0.653 for an agent that does nothing, and that number is not a capability. """ ids = sorted({r['id'] for r in rows}) print('\nSUITE W: completion, harness by rung') print(' ' * 12 + ''.join(t.rjust(9) for t in tiers)) for h in HARNESS_ORDER: hr = [r for r in rows if r['harness'] == h] if not hr: continue comp, rowscore, counts = [], [], [] for t in tiers: cell = [r for r in hr if r['tier'] == t] counts.append(len(cell)) comp.append(mean([r.get('completion') for r in cell])) rowscore.append(mean([r['change'] for r in cell])) print('%-11s %s' % (h, ''.join((fmt(c) + ' ')[:9] for c in comp))) print('%-11s %s' % ('', ''.join((fmt(c) + ' ')[:9] for c in rowscore))) print('%-11s %s' % ('', ''.join((' [%d]' % c).rjust(9) for c in counts))) print('\nSUITE W per scenario, completion') for sid in ids: print(' %s %s' % (sid, next((r['name'] for r in rows if r['id'] == sid), '')[:40])) for h in HARNESS_ORDER: cells = [] for t in tiers: sub = [r for r in rows if r['harness'] == h and r['tier'] == t and r['id'] == sid] cells.append(mean([r.get('completion') for r in sub])) if all(c is None for c in cells): continue print(' %-9s %s' % (h, ''.join((fmt(c) + ' ')[:9] for c in cells))) bad = [r for r in rows if r.get('claimed_done') and (r.get('completion') or 0) < 1.0] print('\nFALSE COMPLETION CLAIMS: %d of %d session(s)' % (len(bad), len(rows))) for r in sorted(bad, key=lambda r: (r['harness'], r['tier'], r['id']))[:20]: print(' %-9s %-6s %-3s completion %.2f' % (r['harness'], r['tier'], r['id'], r.get('completion') or 0)) gates = [r for r in rows if r['regress'] < 1.0] print('\nGATE FAILURES: %d session(s)' % len(gates)) for r in gates[:20]: print(' %-9s %-6s %-3s %s' % (r['harness'], r['tier'], r['id'], r.get('grader_note', '')[:50])) def tool_surface(rows, tiers): """What each harness actually reached for, and how much it repeated itself. Not the declared tool count, which is a property of a manifest, but the distinct tools observed and the calls spent on them. Paper 04 found a harness spending twelve thousand tokens to answer "what is 2+2"; the same behaviour shows up here as a call count with nothing to show for it. """ print('\n' + 'TOOL SURFACE, observed rather than declared') print('%-10s %-6s %8s %9s %9s %s' % ('harness', 'rung', 'distinct', 'calls/sess', 'llm/sess', 'most used')) for h in HARNESS_ORDER: for t in tiers: cell = [r for r in rows if r['harness'] == h and r['tier'] == t] if not cell: continue hist = {} for r in cell: for name, n in (r['metrics'].get('tool_histogram') or {}).items(): hist[name] = hist.get(name, 0) + n top = ', '.join('%s x%d' % (k, v) for k, v in sorted(hist.items(), key=lambda kv: -kv[1])[:3]) print('%-10s %-6s %8d %9.1f %9.1f %s' % ( h, t, len(hist), mean([r['metrics']['tool_calls'] for r in cell]), mean([r['metrics']['llm_calls'] for r in cell]), top[:60])) def variance(rows, tiers): """What changed between identical runs. A single repetition cannot tell a finding from a coincidence, and a mean of three hides which three. This prints the per-repetition scores for every cell that has more than one, and the spread between the best and worst. A 2 of 2 or 0 of 2 statement in the paper is only allowed where this table shows it, and the per repetition scores are printed rather than summarised so a reader can see a cell that disagreed with itself. """ print('\n' + 'BETWEEN REPETITIONS, per cell') print('%-10s %-6s %-4s %s' % ('harness', 'rung', 'scen', 'scores by repetition')) any_multi = False for h in HARNESS_ORDER: for t in tiers: for sid in sorted({r['id'] for r in rows}): cell = sorted([r for r in rows if r['harness'] == h and r['tier'] == t and r['id'] == sid], key=lambda r: r['rep']) if len(cell) < 2: continue any_multi = True vals = [r['change'] for r in cell] spread = max(vals) - min(vals) flag = ' <- varies' if spread > 0.001 else '' print('%-10s %-6s %-4s %s spread %.3f%s' % (h, t, sid, ' '.join('%.3f' % v for v in vals), spread, flag)) if not any_multi: print(' no cell has more than one repetition yet') def main(argv=None): p = argparse.ArgumentParser() p.add_argument('--suite', default='V') p.add_argument('--tier', default='T0,T1,T2') args = p.parse_args(argv) tiers = [t.strip() for t in args.tier.split(',') if t.strip()] rows = load(args.suite) if not rows: print('no rows for suite %s' % args.suite) return 1 bad = [r for r in rows if r.get('valid') is False] rows = [r for r in rows if r.get('valid') is not False] if bad: print('INVALID, excluded from every mean below, %d session(s):' % len(bad)) for r in bad: err = ((r['metrics']['errors'] or [''])[0])[:70] print(' %-9s %-6s %-3s rep%s %s' % (r['harness'], r['tier'], r['id'], r['rep'], err)) print('sessions: %d harnesses: %s' % ( len(rows), ', '.join(sorted({r['harness'] for r in rows})))) floors = floor_table(rows) if args.suite == 'W': w_table(rows, tiers) decompose(rows, floors, tiers) economy(rows, tiers) tool_surface(rows, tiers) else: matrix(rows, floors, tiers) decompose(rows, floors, tiers) per_axis(rows, floors, tiers) economy(rows, tiers) tool_surface(rows, tiers) variance(rows, tiers) regress_note(rows, tiers) return 0 if __name__ == '__main__': raise SystemExit(main())