# -*- coding: utf-8 -*-
"""NEGATIVE CONTROL for the windows-agent benchmark.

Paper 05 section 09 records the absence of this as the weakest point in the
instrument: nothing demonstrated that the graders FAIL an agent that does nothing.
Without it a score of 97.6 is consistent with lenient grading.

This grades three synthetic agents against all 30 real benchmarks, with ZERO API
calls, and asserts what each should score:

  null      does nothing at all                      -> must score ~0
  babbler   answers plausibly, calls no tool          -> must score low
  flailer   calls every tool with junk arguments      -> must not out-score a real run

The suite is only meaningful if the first two score near zero. Any check that a
do-nothing agent passes is a check that measures nothing.
"""
import io
import json
import os
import sys

BENCH = r'C:\xampp\htdocs\windows-agent\bench'
sys.path.insert(0, BENCH)
sys.path.insert(0, r'C:\xampp\htdocs\windows-agent')
os.chdir(BENCH)

import run_bench as RB  # noqa: E402

benchmarks = json.load(io.open(os.path.join(BENCH, 'benchmarks.json'), encoding='utf-8'))['benchmarks']

PLAUSIBLE = ('This machine is running Windows 11 Pro. I checked the system information and '
             'the telemetry, registry and service settings you asked about. The catalogue is '
             'stored as JSON. I have completed the task and verified the result.')


def null_run():
    return RB.Run()


def babbler_run():
    r = RB.Run()
    r.assistant_text = [PLAUSIBLE]
    r.transcript = [PLAUSIBLE]
    r.final_has_content = True
    return r


def flailer_run(b):
    r = RB.Run()
    for name in ('system_info', 'run_powershell', 'read_file', 'write_file', 'registry_get'):
        r.tool_calls.append((name, {'command': 'xyzzy', 'path': 'xyzzy', 'value': 'xyzzy'}))
        r.tool_results.append('xyzzy')
    r.assistant_text = [PLAUSIBLE]
    r.transcript = [PLAUSIBLE]
    r.final_has_content = True
    return r


AGENTS = [('null', lambda b: null_run()),
          ('babbler', lambda b: babbler_run()),
          ('flailer', flailer_run)]

print('=' * 94)
print('NEGATIVE CONTROL - three agents that do no real work, graded by the real graders')
print('=' * 94)
print('  %-10s %8s %8s %9s   %s' % ('agent', 'earned', 'possible', 'percent', 'verdict'))
print('  ' + '-' * 90)

results = {}
for label, make in AGENTS:
    earned = possible = 0
    per_task = []
    for b in benchmarks:
        r = RB.score(b, make(b))
        earned += r['earned']
        possible += r['possible']
        per_task.append((b['id'], r['earned'], r['possible'], r['percent']))
    pct = 100.0 * earned / possible if possible else 0
    results[label] = dict(earned=earned, possible=possible, pct=pct, tasks=per_task)
    verdict = 'OK' if pct < 25 else 'PROBLEM - graders are lenient'
    print('  %-10s %8d %8d %8.1f%%   %s' % (label, earned, possible, pct, verdict))

print()
print('=' * 94)
print('WHICH CHECKS A DO-NOTHING AGENT STILL PASSES')
print('=' * 94)
passed = {}
for b in benchmarks:
    r = RB.score(b, null_run())
    for c in r['checks']:
        if c.get('passed'):
            passed[c.get('type', '?')] = passed.get(c.get('type', '?'), 0) + 1
tot = {}
for b in benchmarks:
    for c in b['checks']:
        tot[c.get('type')] = tot.get(c.get('type'), 0) + 1
if passed:
    for k in sorted(passed, key=lambda x: -passed[x]):
        print('  %-24s passes vacuously %3d of %3d' % (k, passed[k], tot.get(k, 0)))
else:
    print('  none - every check fails on a do-nothing agent')

print()
print('=' * 94)
print('TASKS WHERE THE NULL AGENT SCORED ANYTHING')
print('=' * 94)
bad = [(i, e, p, pc) for i, e, p, pc in results['null']['tasks'] if e > 0]
for i, e, p, pc in bad:
    print('  %-5s %d/%d  %.0f%%' % (i, e, p, pc))
if not bad:
    print('  none - the null agent scores zero on every one of the 30 tasks')

print()
out = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'p05_sanity_results.json')
io.open(out, 'w', encoding='utf-8').write(json.dumps(
    {k: {'earned': v['earned'], 'possible': v['possible'], 'percent': round(v['pct'], 2),
         'per_task': v['tasks']} for k, v in results.items()}, indent=1))
print('written: %s' % out)

REAL = 97.6
print()
print('=' * 94)
print('HEADROOM - the number the paper actually needs')
print('=' * 94)
for label in ('null', 'babbler', 'flailer'):
    print('  measured agent %.1f%% against %-8s %.1f%%   -> the suite discriminates by %.1f points'
          % (REAL, label, results[label]['pct'], REAL - results[label]['pct']))