# -*- coding: utf-8 -*- """Map each benchmark run to the commits that preceded it. Turns "v6 regressed" into "v6 regressed, and here is what changed before it". Read-only against the windows-agent repo. """ import datetime import io import json import os import re import subprocess REPO = r'C:\xampp\htdocs\windows-agent' SRC = os.path.join(REPO, 'bench', 'results') out = subprocess.run(['git', '-C', REPO, 'log', '--date=iso-strict', '--pretty=%H\t%ad\t%s'], capture_output=True) commits = [] for line in out.stdout.decode('utf-8', 'replace').splitlines(): if not line.strip(): continue h, d, s = line.split('\t', 2) commits.append((datetime.datetime.fromisoformat(d).replace(tzinfo=None), h[:7], s)) commits.sort(key=lambda c: c[0]) runs = [] for fn in sorted(os.listdir(SRC)): if not fn.endswith('.json'): continue d = json.load(io.open(os.path.join(SRC, fn), encoding='utf-8')) m = re.match(r'(\d{4}-\d{2}-\d{2})_(\d{2})(\d{2})(\d{2})_(.+)\.json', fn) start = datetime.datetime.fromisoformat( '%sT%s:%s:%s' % (m.group(1), m.group(2), m.group(3), m.group(4))) fin = datetime.datetime.fromisoformat(d['finished_at']) v = int(re.search(r'v(\d+)', m.group(5)).group(1)) rs = d['results'] execs = sum(len(t['runs']) if isinstance(t.get('runs'), list) else 1 for t in rs) runs.append(dict(v=v, label=m.group(5), start=start, fin=fin, overall=d['overall'], cost=d['estimated_cost_usd'], execs=execs)) runs.sort(key=lambda r: r['start']) print('=' * 100) print('WHAT CHANGED BEFORE EACH BENCHMARK RUN') print('=' * 100) prev_end = datetime.datetime(2000, 1, 1) for i, r in enumerate(runs): between = [c for c in commits if prev_end < c[0] <= r['start']] delta = '' if i: delta = ' (%+.1f vs v%d)' % (r['overall'] - runs[i - 1]['overall'], runs[i - 1]['v']) print('\nv%-3d %-22s %s score %.1f $%.5f/exec%s' % (r['v'], r['label'], r['start'].strftime('%m-%d %H:%M'), r['overall'], r['cost'] / r['execs'], delta)) if between: for t, h, s in between: print(' %s %s %s' % (t.strftime('%m-%d %H:%M'), h, s[:78])) else: print(' (no commits between the previous run and this one ' '-- run made from uncommitted working-tree changes)') prev_end = r['start'] after = [c for c in commits if c[0] > runs[-1]['start']] if after: print('\nafter the last run:') for t, h, s in after: print(' %s %s %s' % (t.strftime('%m-%d %H:%M'), h, s[:78]))