# -*- coding: utf-8 -*-
"""The number this paper exists for: pages taken per visitor returned.

    python ratio.py

Reads `crawlers.csv` and `referrals.csv`, which carry no personal data, and
computes the ratio per operator. The step that touches raw logs is `extract.py`
and it is deliberately a different script: the headline can be recomputed by
anyone holding the two CSVs, without holding anybody's IP address.

Operators are paired with the referrer they send traffic from, because the pairing
is the whole claim and it is not always obvious: OpenAI crawls under three agents
and refers from one domain, Anthropic crawls under three and refers from another.
An operator with no referrer domain of its own, a general search crawler or a
dataset collector, is reported separately rather than given a divisor it never had.
"""
import collections
import csv
import io
import os

import argparse
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import sitecfg                                                     # noqa: E402

# operator -> the referrer domains that operator sends visitors from
PAIRS = {
    'OpenAI': (['GPTBot', 'OAI-SearchBot', 'ChatGPT-User'],
               ['chatgpt.com', 'openai.com']),
    'Anthropic': (['ClaudeBot', 'Claude-User', 'Claude-SearchBot'], ['claude.ai']),
    'Perplexity': (['PerplexityBot', 'Perplexity-User'], ['perplexity.ai']),
    'Google AI': (['Google-Extended'], ['gemini.google.com', 'bard.google.com']),
    'Microsoft': (['Bingbot'], ['copilot.microsoft.com', 'bing.com/chat']),
    'Apple': (['Applebot-Extended', 'Applebot'], []),
    'Meta': (['meta-externalagent'], []),
    'ByteDance': (['Bytespider'], []),
    'Common Crawl': (['CCBot'], []),
    'Amazon': (['Amazonbot'], []),
    'Google search': (['Googlebot'], []),
    'You.com': (['YouBot'], ['you.com']),
}


SITE = ''


def read(name):
    path = os.path.join(HERE, 'sites', SITE, name)
    if not os.path.exists(path):
        return []
    with io.open(path, encoding='utf-8') as f:
        return list(csv.DictReader(f))


def main():
    global SITE
    ap = argparse.ArgumentParser()
    ap.add_argument('--site', default='broikos.gr',
                    help='the published alias, not the hostname')
    a = ap.parse_args()
    SITE = a.site
    d = sitecfg.describe(SITE)
    print('%s: %s' % (SITE, d.get('what') or ''))
    crawl = read('crawlers.csv')
    refer = read('referrals.csv')
    if not crawl:
        print('no crawlers.csv: run extract.py first')
        return 1

    req = collections.Counter()
    pages = collections.Counter()
    byts = collections.Counter()
    for r in crawl:
        req[r['operator']] += int(r['requests'])
        pages[r['operator']] += int(r.get('pages') or 0)
        byts[r['operator']] += int(r['bytes'])
    # A visitor is a served page view from a browser. Counting every request
    # that merely carried the referrer would credit an engine with a reader it
    # did not send.
    vis = collections.Counter()
    for r in refer:
        vis[r['engine']] += int(r.get('browser') or 0)

    # If the identity check has been run, the headline uses only the requests
    # that survived it. A user agent string is a claim; this is the part of the
    # claim that came from an address the operator publishes.
    ver = {r['operator']: r for r in read('verification.csv')}
    if ver:
        for name in list(pages):
            if name in ver:
                pages[name] = int(ver[name]['verified_pages'])
                req[name] = int(ver[name]['verified_requests'])

    print('identity-checked counts' if ver else
          'AGENT STRINGS AS CLAIMED: run verify_agents.py for the checked counts')
    print('%-14s %9s %9s %10s %8s %14s' %
          ('operator', 'requests', 'pages', 'bytes', 'visitors', 'pages/visitor'))
    print('-' * 70)
    total_req = total_pages = total_vis = 0
    for name, (agents, domains) in PAIRS.items():
        n_req = sum(req[a] for a in agents)
        n_pag = sum(pages[a] for a in agents)
        n_byt = sum(byts[a] for a in agents)
        n_vis = sum(vis[d] for d in domains)
        # A visitor arrived whether or not the crawl that claimed to be this
        # operator survived the identity check. Dropping the row on a zero
        # request count would also drop the visitor out of the denominator and
        # quietly improve the headline.
        if not n_req and not n_vis:
            continue
        if domains:
            total_req += n_req
            total_pages += n_pag
            total_vis += n_vis
            ratio = ('%.0f : 1' % (n_pag / n_vis)) if n_vis else 'no visitor at all'
        else:
            ratio = 'no referrer of its own'
        print('%-14s %9d %9d %8.1f MB %8s %14s'
              % (name, n_req, n_pag, n_byt / 1e6, n_vis if domains else '', ratio))

    print('-' * 70)
    if total_vis:
        print('%-14s %9d %9d %11s %8d %10.0f : 1'
              % ('all pairable', total_req, total_pages, '', total_vis,
                 total_pages / total_vis))
    else:
        print('%-14s %9d requests, %d pages, and not one visitor returned'
              % ('all pairable', total_req, total_pages))

    known = set()
    for agents, _d in PAIRS.values():
        known.update(agents)
    missing = sorted(set(req) - known)
    if missing:
        print('\nnot paired with any operator: %s' % ', '.join(missing))
    return 0


if __name__ == '__main__':
    raise SystemExit(main())