# -*- coding: utf-8 -*-
"""windows-agent longitudinal analysis, normalised per task-execution.

The protocol changed at v10 (3 repetitions per task instead of 1), so every
run-level total must be divided by the number of task-executions before any
version is compared with another.
"""
import collections
import io
import json
import os
import re

SRC = r'C:\xampp\htdocs\windows-agent\bench\results'

runs = []
for fn in sorted(os.listdir(SRC)):
    if not fn.endswith('.json'):
        continue
    d = json.load(io.open(os.path.join(SRC, fn), encoding='utf-8'))
    m = re.match(r'(\d{4}-\d{2}-\d{2})_\d{6}_(.+)\.json', fn)
    v = re.search(r'v(\d+)', m.group(2))
    rs = d['results']
    execs = sum(len(t['runs']) if isinstance(t.get('runs'), list) else 1 for t in rs)
    runs.append(dict(v=int(v.group(1)), label=m.group(2), date=m.group(1),
                     overall=d['overall'], cost=d['estimated_cost_usd'],
                     mins=d['elapsed_seconds'] / 60.0, results=rs, execs=execs,
                     reps=execs // len(rs)))
runs.sort(key=lambda r: r['v'])

print('=' * 104)
print('NORMALISED PER TASK-EXECUTION   (the protocol changed at v10: 3 reps, not 1)')
print('=' * 104)
print('  %-4s %-22s %5s %6s %9s %10s %9s %9s %8s' %
      ('ver', 'label', 'reps', 'score', 'run cost', '$/exec', 'run mins', 's/exec', 'calls/ex'))
print('  ' + '-' * 100)
prev = None
for r in runs:
    tc = sum(t.get('tool_calls') or 0 for t in r['results'])
    d = ''
    if prev:
        dd = r['overall'] - prev['overall']
        d = '  %+.1f' % dd
    print('  v%-3d %-22s %5d %6.1f %9.4f %10.5f %9.1f %9.1f %8.2f%s'
          % (r['v'], r['label'], r['reps'], r['overall'], r['cost'],
             r['cost'] / r['execs'], r['mins'], r['mins'] * 60 / r['execs'],
             tc / r['execs'], d))
    prev = r

print()
print('=' * 104)
print('COST PER POINT EARNED   (the number a buyer actually cares about)')
print('=' * 104)
best = min(runs, key=lambda r: (r['cost'] / r['execs']) / r['overall'])
for r in runs:
    cpp = (r['cost'] / r['execs']) / r['overall'] * 100
    print('  v%-3d %-22s %6.1f pts  $%.5f per 100 points  %s'
          % (r['v'], r['label'], r['overall'], cpp, '<-- best value' if r is best else ''))

print()
print('=' * 104)
print('REGRESSIONS  (versions that scored worse than the one before)')
print('=' * 104)
for a, b in zip(runs, runs[1:]):
    if b['overall'] < a['overall']:
        print('  v%d -> v%d : %.1f -> %.1f  (%+.1f)   cost/exec $%.5f -> $%.5f (%+.0f%%)'
              % (a['v'], b['v'], a['overall'], b['overall'], b['overall'] - a['overall'],
                 a['cost'] / a['execs'], b['cost'] / b['execs'],
                 100 * ((b['cost'] / b['execs']) / (a['cost'] / a['execs']) - 1)))
        # which tasks moved
        pa = {t['id']: t['percent'] for t in a['results']}
        moved = sorted(((t['percent'] - pa.get(t['id'], 0), t['id'], t['name'], t['category'])
                        for t in b['results'] if t['percent'] != pa.get(t['id'])),
                       key=lambda x: x[0])
        for delta, tid, name, cat in moved[:6]:
            print('       %-5s %-42s %-12s %+.0f pts' % (tid, name[:42], cat, delta))

print()
print('=' * 104)
print('TASKS THAT NEVER REACHED 100 IN ANY VERSION')
print('=' * 104)
bytask = collections.defaultdict(dict)
for r in runs:
    for t in r['results']:
        bytask[t['id']][r['v']] = t['percent']
never = [(tid, d) for tid, d in bytask.items() if max(d.values()) < 100]
for tid, d in sorted(never):
    nm = next(t['name'] for t in runs[-1]['results'] if t['id'] == tid)
    cat = next(t['category'] for t in runs[-1]['results'] if t['id'] == tid)
    print('  %-5s %-44s %-12s best %.0f  latest %.0f'
          % (tid, nm[:44], cat, max(d.values()), d[runs[-1]['v']]))
print('  %d of %d tasks never scored 100' % (len(never), len(bytask)))

print()
print('=' * 104)
print('ERROR TAXONOMY  (recorded error strings across all versions)')
print('=' * 104)
errs = collections.Counter()
for r in runs:
    for t in r['results']:
        e = (t.get('error') or '').strip()
        if e:
            errs[(re.sub(r'\d+', 'N', e)[:70], t['category'])] += 1
for (e, cat), n in errs.most_common():
    print('  x%-2d [%-11s] %s' % (n, cat, e))
if not errs:
    print('  none recorded')