# -*- coding: utf-8 -*-
"""Generate the paper's tables as markdown, straight from the logs.

    python tables.py            writes draft/tables/*.md

Every number in the draft comes from a file this script writes. Nothing is typed
into the prose by hand, because the failure this programme keeps repeating is not
a wrong calculation, it is a correct number that ends up on the wrong page: three
papers once shipped carrying another paper's cover statistics, and every audit
passed because each number was right somewhere.

Each generated file carries the count of sessions behind it and the date it was
generated, so a table that has gone stale says so on its face.
"""
import glob
import io
import json
import os
import sys
from datetime import datetime, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
LOGS = os.path.join(HERE, 'logs')
OUT = os.path.abspath(os.path.join(HERE, '..', 'draft', 'tables'))

import analyse as A                                         # noqa: E402

LABEL = {'T0': 'local 8B', 'T1': 'flash', 'T2': 'pro', 'T3': 'luna, reasoning max',
         'S-off': 'flash, thinking off', 'S-high': 'flash, thinking high'}
NICE = {'axium': 'Axium', 'axium-guard': 'Axium with the write guard', 'orange': 'Orange', 'winagent': 'windows-agent',
        'hermes': 'Hermes', 'dsh': 'DeepSeek Harness', 'openclaw': 'OpenClaw',
        'null': 'null control', 'prose': 'prose control'}


def stamp(n):
    return ('\n*%d sessions, generated %s.*\n'
            % (n, datetime.now(timezone.utc).strftime('%d %B %Y')))


def cell(v, nd=3):
    return 'not run' if v is None else '%.*f' % (nd, v)


def write(name, body):
    os.makedirs(OUT, exist_ok=True)
    with io.open(os.path.join(OUT, name), 'w', encoding='utf-8') as f:
        f.write(body)
    print('wrote %s' % os.path.join('draft', 'tables', name))


def floor_table(rows, suite):
    ctrl = [r for r in rows if A.PARTY.get(r['harness']) == 'control']
    ids = sorted({r['id'] for r in rows})
    out = ['| scenario | axis | does nothing | says it did it |', '|---|---|---|---|']
    for sid in ids:
        n = A.mean([r['change'] for r in ctrl
                    if r['id'] == sid and r['harness'] == 'null'])
        p = A.mean([r['change'] for r in ctrl
                    if r['id'] == sid and r['harness'] == 'prose'])
        axis = next((r['axis'] for r in rows if r['id'] == sid), '')
        out.append('| %s | %s | %s | %s |' % (sid, axis, cell(n), cell(p)))
    return '\n'.join(out) + '\n' + stamp(len(ctrl))


def matrix_table(rows, tiers, floors):
    hs = [h for h in A.HARNESS_ORDER if A.PARTY.get(h) != 'control'
          and any(r['harness'] == h for r in rows)]
    head = '| harness | ' + ' | '.join('%s (%s)' % (t, LABEL.get(t, t))
                                       for t in tiers) + ' |'
    out = [head, '|---' * (len(tiers) + 1) + '|']
    for h in hs:
        cells = []
        for t in tiers:
            sub = [r for r in rows if r['harness'] == h and r['tier'] == t]
            adj = A.mean([A.adjusted(r['change'], floors.get(r['id'])) for r in sub])
            raw = A.mean([r['change'] for r in sub])
            cells.append('not run' if adj is None
                         else '%s (%s raw, n=%d)' % (cell(adj), cell(raw, 2), len(sub)))
        out.append('| %s | %s |' % (NICE.get(h, h), ' | '.join(cells)))
    # Count only what the table shows. The side study runs two further rungs, and
    # a stamp that counted those would credit this table with sessions it does
    # not display.
    n = len([r for r in rows if A.PARTY.get(r['harness']) != 'control'
             and r['tier'] in tiers])
    return '\n'.join(out) + '\n' + stamp(n)


def economy_table(rows, tiers):
    hs = [h for h in A.HARNESS_ORDER if A.PARTY.get(h) != 'control'
          and any(r['harness'] == h for r in rows)]
    out = ['| harness | rung | USD per session | input tokens | output tokens '
           '| wall seconds | tool calls |',
           '|---|---|---|---|---|---|---|']
    n = 0
    for h in hs:
        for t in tiers:
            sub = [r for r in rows if r['harness'] == h and r['tier'] == t]
            if not sub:
                continue
            n += len(sub)
            m = [r['metrics'] for r in sub]
            priced = any(r.get('priced') for r in sub)
            usd = A.mean([r.get('cost_usd_recomputed', r['metrics']['cost_usd'])
                          for r in sub])
            out.append('| %s | %s | %s | %s | %s | %s | %s |' % (
                NICE.get(h, h), LABEL.get(t, t),
                ('%.5f' % usd) if priced else 'no price',
                '{:,}'.format(int(A.mean([x['input_tokens'] for x in m]))),
                '{:,}'.format(int(A.mean([x['output_tokens'] for x in m]))),
                int(A.mean([x['wall_s'] for x in m])),
                '%.1f' % A.mean([x['tool_calls'] for x in m])))
    return '\n'.join(out) + '\n' + stamp(n)


def asked_table(rows, tiers):
    out = ['| harness | rung | asked before acting | tools in the destructive turn |',
           '|---|---|---|---|']
    n = 0
    for r in sorted(rows, key=lambda r: (A.HARNESS_ORDER.index(r['harness'])
                                         if r['harness'] in A.HARNESS_ORDER else 99,
                                         r['tier'])):
        if r['id'] != 'V4' or A.PARTY.get(r['harness']) == 'control':
            continue
        if r['tier'] not in tiers:
            continue
        n += 1
        d = dict((k, v) for k, v in r['change_detail'])
        asked = d.get('pushed back or asked first')
        t1 = (r['turns'] or [{}])[0]
        out.append('| %s | %s | %s | %s |' % (
            NICE.get(r['harness'], r['harness']), LABEL.get(r['tier'], r['tier']),
            'yes' if asked else 'no',
            ', '.join((t1.get('tools') or [])[:4]) or 'none'))
    return '\n'.join(out) + '\n' + stamp(n)


def w_table(rows, tiers):
    hs = [h for h in A.HARNESS_ORDER if A.PARTY.get(h) != 'control'
          and any(r['harness'] == h for r in rows)]
    out = ['| harness | ' + ' | '.join(LABEL.get(t, t) for t in tiers) + ' |',
           '|---' * (len(tiers) + 1) + '|']
    n = 0
    for h in hs:
        cells = []
        for t in tiers:
            sub = [r for r in rows if r['harness'] == h and r['tier'] == t]
            n += len(sub)
            v = A.mean([r.get('completion') for r in sub])
            cells.append('not run' if v is None
                         else '%s (n=%d)' % (cell(v, 2), len(sub)))
        out.append('| %s | %s |' % (NICE.get(h, h), ' | '.join(cells)))
    return '\n'.join(out) + '\n' + stamp(n)



def decomposition_table(rows, tiers, floors):
    """How much of the spread is the model, how much the harness, how much neither.

    Balanced grid only: a harness missing a cell is named under the table rather
    than averaged into it, because an unbalanced decomposition does not add to
    the whole.
    """
    all_ids = sorted({r['id'] for r in rows})
    cands = [h for h in A.HARNESS_ORDER if A.PARTY.get(h) != 'control'
             and any(r['harness'] == h for r in rows)]

    def full(h, t):
        ids = {r['id'] for r in rows if r['harness'] == h and r['tier'] == t}
        return set(all_ids) <= ids

    hs = [h for h in cands if all(full(h, t) for t in tiers)]
    dropped = [h for h in cands if h not in hs]
    cells = {}
    for h in hs:
        for t in tiers:
            sub = [r for r in rows if r['harness'] == h and r['tier'] == t]
            v = A.mean([A.adjusted(r['change'], floors.get(r['id'])) for r in sub])
            if v is not None:
                cells[(h, t)] = v
    if len(cells) < 4:
        return '*Not enough complete cells to decompose yet.*\n'
    grand = A.mean(list(cells.values()))
    h_means = {h: A.mean([cells[(h, t)] for t in tiers if (h, t) in cells]) for h in hs}
    t_means = {t: A.mean([cells[(h, t)] for h in hs if (h, t) in cells]) for t in tiers}
    ss_total = sum((v - grand) ** 2 for v in cells.values())
    ss_h = sum((h_means[h] - grand) ** 2 for (h, t) in cells)
    ss_t = sum((t_means[t] - grand) ** 2 for (h, t) in cells)
    ss_i = sum((cells[(h, t)] - h_means[h] - t_means[t] + grand) ** 2 for (h, t) in cells)

    out = ['| source of the spread | share |', '|---|---|']
    for label, ss in (('the model, moving between rungs', ss_t),
                      ('the harness, moving between columns', ss_h),
                      ('their interaction', ss_i)):
        out.append('| %s | %.1f%% |' % (label, 100 * ss / ss_total if ss_total else 0))
    out.append('')
    out.append('| rung | mean, floor adjusted |')
    out.append('|---|---|')
    for t in tiers:
        if t in t_means and t_means[t] is not None:
            out.append('| %s (%s) | %.3f |' % (t, LABEL.get(t, t), t_means[t]))
    out.append('')
    out.append('| harness | mean, floor adjusted |')
    out.append('|---|---|')
    for h in hs:
        out.append('| %s | %.3f |' % (NICE.get(h, h), h_means[h]))
    if dropped:
        out.append('')
        out.append('*Excluded for an incomplete cell: %s.*'
                   % ', '.join(NICE.get(h, h) for h in dropped))
    n = sum(len([r for r in rows if r['harness'] == h and r['tier'] == t])
            for (h, t) in cells)
    return '\n'.join(out) + '\n' + stamp(n)


def repetition_table(rows, tiers, floors):
    """What changed between identical runs, per cell, so a mean cannot hide it."""
    hs = [h for h in A.HARNESS_ORDER if A.PARTY.get(h) != 'control'
          and any(r['harness'] == h for r in rows)]
    out = ['| harness | rung | repetition 1 | repetition 2 | spread |',
           '|---|---|---|---|---|']
    n = 0
    for h in hs:
        for t in tiers:
            cell = sorted([r for r in rows if r['harness'] == h and r['tier'] == t],
                          key=lambda r: r['rep'])
            if not cell:
                continue
            by_rep = {}
            for r in cell:
                by_rep.setdefault(r['rep'], []).append(r['change'])
            if len(by_rep) < 2:
                continue
            means = [A.mean(by_rep[k]) for k in sorted(by_rep)]
            n += len(cell)
            out.append('| %s | %s | %s | %s | %.3f |' % (
                NICE.get(h, h), LABEL.get(t, t), cell_fmt(means[0]),
                cell_fmt(means[1]), abs(means[0] - means[1])))
    if len(out) == 2:
        return '*No cell has two repetitions yet.*\n'
    return '\n'.join(out) + '\n' + stamp(n)


def cell_fmt(v):
    return 'not run' if v is None else '%.3f' % v



def side_study_table(rows, floors):
    """Thinking off against thinking high, at fixed weights."""
    tiers = ['S-off', 'S-high']
    hs = [h for h in A.HARNESS_ORDER if A.PARTY.get(h) != 'control'
          and any(r['harness'] == h and r['tier'] in tiers for r in rows)]
    if not hs:
        return '*The side study has not run yet.*\n'
    out = ['| harness | thinking off | thinking high | difference |',
           '|---|---|---|---|']
    n = 0
    for h in hs:
        vals = []
        for t in tiers:
            sub = [r for r in rows if r['harness'] == h and r['tier'] == t]
            n += len(sub)
            vals.append(A.mean([A.adjusted(r['change'], floors.get(r['id']))
                                for r in sub]))
        diff = ('' if None in vals else '%+.3f' % (vals[1] - vals[0]))
        out.append('| %s | %s | %s | %s |' % (NICE.get(h, h), cell_fmt(vals[0]),
                                              cell_fmt(vals[1]), diff or 'not run'))
    return '\n'.join(out) + '\n' + stamp(n)



def damage_table(rows, tiers):
    """Sessions that failed a scenario gate, which on the coding suite means the
    agent destroyed something it was never asked to touch."""
    hs = [h for h in A.HARNESS_ORDER if A.PARTY.get(h) != 'control'
          and any(r['harness'] == h for r in rows)]
    out = ['| harness | ' + ' | '.join(LABEL.get(t, t) for t in tiers) +
           ' | total |', '|---' * (len(tiers) + 2) + '|']
    n = 0
    grand = 0
    for h in hs:
        cells, total = [], 0
        for t in tiers:
            sub = [r for r in rows if r['harness'] == h and r['tier'] == t]
            n += len(sub)
            bad = sum(1 for r in sub if r['regress'] < 1.0)
            total += bad
            cells.append(str(bad) if sub else 'not run')
        grand += total
        out.append('| %s | %s | %d |' % (NICE.get(h, h), ' | '.join(cells), total))
    out.append('| **all** | %s | **%d** |'
               % (' | '.join(str(sum(1 for r in rows if r['tier'] == t
                                     and r['regress'] < 1.0)) for t in tiers), grand))
    return '\n'.join(out) + '\n' + stamp(n)



def headline_table(v_rows, w_rows, tiers, floors):
    """Every figure the summary quotes, computed here rather than typed there.

    Restricted to the primary rungs. The side study runs two more rungs on four of
    the harnesses, and folding those into a headline would quietly count some
    harnesses twice.
    """
    v_rows = [r for r in v_rows if r['tier'] in tiers]
    w_rows = [r for r in w_rows if r['tier'] in tiers]

    def cells(rows, tier):
        return [r for r in rows if r['tier'] == tier]

    hs = [h for h in A.HARNESS_ORDER if A.PARTY.get(h) != 'control'
          and any(r['harness'] == h for r in v_rows)]
    rung_mean = {}
    for t in tiers:
        rung_mean[t] = A.mean([A.adjusted(r['change'], floors.get(r['id']))
                               for r in cells(v_rows, t)])
    harness_mean = {}
    for h in hs:
        harness_mean[h] = A.mean([A.adjusted(r['change'], floors.get(r['id']))
                                  for r in v_rows if r['harness'] == h])
    best = max(harness_mean, key=lambda h: harness_mean[h])
    worst = min(harness_mean, key=lambda h: harness_mean[h])

    comp = {}
    for t in tiers:
        comp[t] = A.mean([r.get('completion') for r in cells(w_rows, t)])
    damage = sum(1 for r in w_rows if r['regress'] < 1.0)
    damage_hosted = sum(1 for r in w_rows if r['regress'] < 1.0 and r['tier'] != 'T0')
    clean = [h for h in hs
             if not any(r['regress'] < 1.0 for r in w_rows if r['harness'] == h)]

    rows = [
        ('sessions behind these figures',
         '%d behaviour, %d coding' % (len(v_rows), len(w_rows))),
        ('floor-adjusted mean, local 8B', '%.3f' % (rung_mean.get('T0') or 0)),
        ('floor-adjusted mean, flash', '%.3f' % (rung_mean.get('T1') or 0)),
        ('floor-adjusted mean, pro', '%.3f' % (rung_mean.get('T2') or 0)),
        ('floor-adjusted mean, luna at max reasoning',
         '%.3f' % (rung_mean.get('T3') or 0)),
        ('best harness, all rungs', '%s, %.3f' % (NICE.get(best, best),
                                                  harness_mean[best])),
        ('weakest harness, all rungs', '%s, %.3f' % (NICE.get(worst, worst),
                                                     harness_mean[worst])),
        ('coding completion, local 8B', '%.2f' % (comp.get('T0') or 0)),
        ('coding completion, luna at max reasoning', '%.2f' % (comp.get('T3') or 0)),
        ('sessions that destroyed user files', '%d, of which %d at a hosted model'
         % (damage, damage_hosted)),
        ('harnesses that never destroyed user files',
         ', '.join(NICE.get(h, h) for h in clean) or 'none'),
    ]
    out = ['| figure | value |', '|---|---|']
    out += ['| %s | %s |' % r for r in rows]
    return '\n'.join(out) + '\n' + stamp(len(v_rows) + len(w_rows))


def main():
    tiers = ['T0', 'T1', 'T2', 'T3']
    v = [r for r in A.load('V') if r.get('valid') is not False]
    floors = {}
    ctrl = [r for r in v if A.PARTY.get(r['harness']) == 'control']
    for sid in sorted({r['id'] for r in v}):
        floors[sid] = A.mean([r['change'] for r in ctrl
                              if r['id'] == sid and r['harness'] == 'null'])
    write('floor-v.md', floor_table(v, 'V'))
    write('matrix-v.md', matrix_table(v, tiers, floors))
    write('economy-v.md', economy_table(v, tiers))
    write('asked-first.md', asked_table(v, tiers))
    write('decomposition-v.md', decomposition_table(v, tiers, floors))
    write('repetitions-v.md', repetition_table(v, tiers, floors))
    write('side-study.md', side_study_table(v, floors))

    w = [r for r in A.load('W') if r.get('valid') is not False]
    w_real = [r for r in w if A.PARTY.get(r['harness']) != 'control']
    v_real = [r for r in v if A.PARTY.get(r['harness']) != 'control']
    if w_real:
        write('headline.md', headline_table(v_real, w_real, tiers, floors))
    if any(A.PARTY.get(r['harness']) != 'control' for r in w):
        write('floor-w.md', floor_table(w, 'W'))
        write('completion-w.md', w_table(w, tiers))
        write('damage-w.md', damage_table(w, tiers))
    else:
        print('suite W: controls only so far, harness tables not written')
    return 0


if __name__ == '__main__':
    raise SystemExit(main())