# -*- coding: utf-8 -*-
"""Provenance checks over the logs. Run before any figure is written.

Paper 04's audits recomputed every headline number from the published data and
confirmed it appeared in the rendered PDF, and three papers still shipped with
another paper's statistics on the cover, because "this number is correct and it
is on the page" was true and the check never asked whether the number belonged
there.

So this file checks identity rather than value:

  1. Every row's agent matches the file it is in, and the harness in its manifest.
  2. Every row's model matches the rung it claims, and the adapter label names
     that model.
  3. No (harness, rung, suite, scenario, repetition) key appears twice.
  4. No row carries a cost at a rung with no price, and none carries a price at a
     rung that has one without the rate it was billed at.
  5. Every row's scenario belongs to the suite its file names.
  6. A control row names no model at all.

Exit code 1 if anything fails, so it can gate a build.
"""
import glob
import io
import json
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
LOGS = os.path.join(HERE, 'logs')

import tiers as TI                                          # noqa: E402

SUITE_IDS = {'V': {'V1', 'V2', 'V3', 'V4', 'V5'},
             'W': {'W1', 'W2', 'W3', 'W4', 'W5', 'W6'}}
CONTROLS = {'null', 'prose'}


def check():
    problems = []
    seen = {}
    files = sorted(glob.glob(os.path.join(LOGS, '*.jsonl')))
    files = [f for f in files if 'pre-correction' not in f]
    total = 0
    for path in files:
        base = os.path.basename(path)[:-len('.jsonl')]
        parts = base.split('__')
        if len(parts) != 3:
            problems.append('%s: filename is not harness__tier__suite' % base)
            continue
        f_harness, f_tier, f_suite = parts
        for n, line in enumerate(io.open(path, encoding='utf-8'), 1):
            line = line.strip()
            if not line:
                continue
            total += 1
            r = json.loads(line)
            where = '%s:%d' % (base, n)

            if r.get('agent') != f_harness or r.get('harness') != f_harness:
                problems.append('%s: agent %r / harness %r in a %s file'
                                % (where, r.get('agent'), r.get('harness'), f_harness))
            if r.get('tier') != f_tier:
                problems.append('%s: row rung %r in a %s file' % (where, r.get('tier'),
                                                                  f_tier))
            if r.get('suite') and r.get('suite') != f_suite:
                problems.append('%s: row suite %r in a %s file' % (where,
                                                                   r.get('suite'), f_suite))
            if r.get('id') not in SUITE_IDS.get(f_suite, set()):
                problems.append('%s: scenario %r does not belong to suite %s'
                                % (where, r.get('id'), f_suite))

            key = (f_harness, f_tier, f_suite, r.get('id'), r.get('rep'))
            if key in seen:
                problems.append('%s: duplicate of %s' % (where, seen[key]))
            seen[key] = where

            if f_harness in CONTROLS:
                if r.get('model') not in (None, 'none'):
                    problems.append('%s: a control names model %r' % (where,
                                                                      r.get('model')))
                if r['metrics']['llm_calls'] or r['metrics']['cost_usd']:
                    problems.append('%s: a control made %s call(s) costing %s'
                                    % (where, r['metrics']['llm_calls'],
                                       r['metrics']['cost_usd']))
                continue

            spec = TI.TIERS.get(f_tier)
            if not spec:
                problems.append('%s: unknown rung %s' % (where, f_tier))
                continue
            if r.get('model') != spec['model']:
                problems.append('%s: model %r on rung %s, which is %s'
                                % (where, r.get('model'), f_tier, spec['model']))
            label = r.get('label') or ''
            if spec['model'] not in label and r.get('adapter_label', '') and \
                    spec['model'] not in r.get('adapter_label', ''):
                problems.append('%s: label %r does not name %s'
                                % (where, label[:60], spec['model']))
            if not spec['priced'] and r['metrics']['cost_usd']:
                problems.append('%s: a cost of %s at an unpriced rung'
                                % (where, r['metrics']['cost_usd']))

    print('integrity: %d row(s) in %d file(s), %d problem(s)'
          % (total, len(files), len(problems)))
    for p in problems:
        print('  !!', p)
    return not problems


if __name__ == '__main__':
    raise SystemExit(0 if check() else 1)