# -*- coding: utf-8 -*-
"""Turn raw access logs into the two things paper 09 measures, and nothing else.

    python extract.py broikos.gr

Reads the gzipped archives in `raw/<site>/`, and writes aggregates only:

    crawlers.csv   requests, bytes and distinct paths per operator per day
    referrals.csv  sessions arriving from each answer engine per day
    format.md      what the log line actually contains, checked rather than assumed

No IP address, no user agent string and no raw line leaves this script. The ratio
the paper reports, pages taken per visitor returned, is computed from the two CSVs
by `ratio.py`, so the step that produces the headline is separable from the step
that touches personal data.

The operator list is the published set of AI crawlers and the answer-engine
referrers as of August 2026. Anything not on it is counted as `other`, which is
reported rather than dropped: a crawler this study does not know about is a gap in
the study, not an absence in the world.
"""
import argparse
import collections
import csv
import glob
import gzip
import io
import os
import re
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
RAW = os.path.join(HERE, 'raw')

sys.path.insert(0, HERE)
import sitecfg                                                     # noqa: E402

CRAWLERS = [
    ('GPTBot', r'GPTBot'),
    ('OAI-SearchBot', r'OAI-SearchBot'),
    ('ChatGPT-User', r'ChatGPT-User'),
    ('ClaudeBot', r'ClaudeBot'),
    ('Claude-User', r'Claude-User'),
    ('Claude-SearchBot', r'Claude-SearchBot'),
    ('PerplexityBot', r'PerplexityBot'),
    ('Perplexity-User', r'Perplexity-User'),
    ('Google-Extended', r'Google-Extended'),
    ('Googlebot', r'Googlebot'),
    ('Bingbot', r'bingbot'),
    ('Bytespider', r'Bytespider'),
    ('Amazonbot', r'Amazonbot'),
    ('Applebot-Extended', r'Applebot-Extended'),
    ('Applebot', r'Applebot(?!-Extended)'),
    ('meta-externalagent', r'meta-externalagent'),
    ('CCBot', r'CCBot'),
    ('DuckAssistBot', r'DuckAssistBot'),
    ('YouBot', r'YouBot'),
    ('cohere-ai', r'cohere-ai'),
]

REFERRERS = [
    ('chatgpt.com', r'chatgpt\.com'),
    ('openai.com', r'openai\.com'),
    ('perplexity.ai', r'perplexity\.ai'),
    ('gemini.google.com', r'gemini\.google\.com'),
    ('bard.google.com', r'bard\.google\.com'),
    ('copilot.microsoft.com', r'copilot\.microsoft\.com'),
    ('bing.com/chat', r'bing\.com/chat'),
    ('claude.ai', r'claude\.ai'),
    ('you.com', r'you\.com'),
    # duck.ai, not duckduckgo.com. The search engine refers from the second and
    # its assistant from the first, and only one of them is an answer engine.
    ('duck.ai', r'duck\.ai'),
]

CRAWLER_RE = [(n, re.compile(p, re.I)) for n, p in CRAWLERS]
REFERRER_RE = [(n, re.compile(p, re.I)) for n, p in REFERRERS]

# A stylesheet is not a page. The headline claim is pages taken per visitor
# returned, and a crawler that fetches one article plus its twelve images has
# taken one page, not thirteen. Both counts are kept, because the raw request
# count is what a bandwidth bill sees and the page count is what the argument
# about content is actually about.
ASSET = re.compile(r'\.(?:css|js|mjs|png|jpe?g|gif|svg|webp|avif|ico|woff2?|ttf|'
                   r'eot|map|mp4|webm|mp3|zip|gz)(?:$|\?)', re.I)


def is_page(path):
    return not ASSET.search(path or '')


# "Not a crawler I listed" is not the same as "a person". Uptime monitors,
# vulnerability scanners, feed readers and libraries all arrive without a browser
# in their agent string, and counting them as human would inflate the very
# denominator this paper's headline is compared against.
BROWSER = re.compile(r'Mozilla/\d', re.I)
BROWSER_ENGINE = re.compile(r'(?:Chrome|Firefox|Safari|Edg|OPR|Trident)/', re.I)
NOT_HUMAN = re.compile(r'bot|crawl|spider|scrap|http|python|curl|wget|okhttp|'
                       r'java|go-http|libwww|scan|monitor|uptime|preview|fetch|'
                       r'headless|phantom|slurp|feed|rss|probe|check', re.I)


def looks_human(agent):
    return bool(BROWSER.search(agent) and BROWSER_ENGINE.search(agent)
                and not NOT_HUMAN.search(agent))


def _host(ref, site_host=''):
    """The referring site, or a word for why there is none.

    Only the host is kept. A full referrer URL can carry a search term, and a
    search term can identify a person, which is the same reason the addresses
    never leave this script.

    The site's own host is replaced here rather than downstream. One of the two
    sites in this study is anonymous, and internal navigation is the single
    largest referrer on any site, so a self-referral written out in full would
    put the very hostname the study is protecting at the top of a published
    table. It very nearly did.
    """
    if not ref or ref == '-':
        return '(none: typed, bookmarked or hidden)'
    m = re.match(r'https?://([^/:]+)', ref)
    host = (m.group(1) if m else ref).lower()
    if host.startswith('www.'):
        host = host[4:]
    # A referrer that is a bare address is masked like every other address here.
    # It is usually a server referring to itself, but it can also be somebody's
    # own proxy, and the rule in this paper is that no address is published.
    if re.match(r'^(?:\d{1,3}\.){3}\d{1,3}$', host) or ':' in host:
        return '(an address rather than a hostname)'
    if site_host:
        label = site_host.split('.')[0]
        # Not just the exact host. A cPanel preview URL and a staging subdomain
        # both carry the site's name inside somebody else's domain, and both got
        # past an equality test the first time this was written.
        if (site_host in host or host in site_host
                or label in host.replace('-', '.').split('.')):
            return '(this site)'
    return host

# Combined log: host ident user [date] "request" status bytes "referer" "agent"
LINE = re.compile(
    r'^(?P<host>\S+) \S+ \S+ \[(?P<when>[^\]]+)\] "(?P<req>[^"]*)" '
    r'(?P<status>\d{3}) (?P<bytes>\S+)(?: "(?P<ref>[^"]*)" "(?P<agent>[^"]*)")?')


def read_lines(site):
    for path in sorted(glob.glob(os.path.join(RAW, site, '*.gz'))):
        with gzip.open(path, 'rt', encoding='utf-8', errors='replace') as f:
            for line in f:
                yield os.path.basename(path), line


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('site', nargs='?', default='broikos.gr')
    a = ap.parse_args()
    # From here on the host is replaced by its published alias. One site is
    # named and one is not, and the difference must not survive into a filename.
    al = sitecfg.alias(a.site)
    out_dir = sitecfg.outdir(al)

    crawl = collections.defaultdict(
        lambda: {'req': 0, 'ok': 0, 'pages': 0, 'bytes': 0, 'paths': set()})
    refer = collections.defaultdict(lambda: {'req': 0, 'pages': 0,
                                             'browser': 0})
    other = collections.Counter()
    # How big the site actually is, from the site's own side: distinct page
    # paths anybody was served during the window. Without it, "took 1,069
    # pages" has no scale to be read against.
    site_pages = set()
    parsed = unparsed = human = 0
    have_ref = have_agent = 0
    files = set()

    for fname, line in read_lines(a.site):
        files.add(fname)
        m = LINE.match(line)
        if not m:
            unparsed += 1
            continue
        parsed += 1
        agent = m.group('agent') or ''
        ref = m.group('ref') or ''
        if agent:
            have_agent += 1
        if ref and ref != '-':
            have_ref += 1
        day = (m.group('when') or '')[:11]
        path = (m.group('req') or ' ').split(' ')[1] if ' ' in (m.group('req') or '') \
            else '-'
        try:
            size = int(m.group('bytes'))
        except (TypeError, ValueError):
            size = 0
        try:
            status = int(m.group('status'))
        except (TypeError, ValueError):
            status = 0
        served = status in (200, 206)
        if served and is_page(path):
            site_pages.add(path.split('?')[0][:200])

        hit = None
        for name, rx in CRAWLER_RE:
            if rx.search(agent):
                hit = name
                break
        if hit:
            row = crawl[(day, hit)]
            row['req'] += 1
            row['bytes'] += size
            row['paths'].add(path[:120])
            if served:
                row['ok'] += 1
                if is_page(path):
                    row['pages'] += 1
            continue
        human += 1
        # The site's own baseline. Two arrivals from an answer engine means one
        # thing on a site nobody visits and another on a site with real traffic,
        # so the denominator is measured rather than left to the reader.
        if served and is_page(path):
            other[(day[3:], 'browser' if looks_human(agent) else 'other',
                   _host(ref, a.site))] += 1
        for name, rx in REFERRER_RE:
            if rx.search(ref):
                row = refer[(day, name)]
                row['req'] += 1
                if served and is_page(path):
                    row['pages'] += 1
                    # An arrival carrying an answer engine's referrer from a
                    # library rather than a browser is not a reader, and the
                    # paper's divisor is readers.
                    if looks_human(agent):
                        row['browser'] += 1
                break

    with io.open(os.path.join(out_dir, 'crawlers.csv'), 'w', encoding='utf-8',
                 newline='') as f:
        w = csv.writer(f)
        w.writerow(['day', 'operator', 'requests', 'served', 'pages', 'bytes',
                    'distinct_paths'])
        for (day, name), row in sorted(crawl.items()):
            w.writerow([day, name, row['req'], row['ok'], row['pages'],
                        row['bytes'], len(row['paths'])])

    with io.open(os.path.join(out_dir, 'referrals.csv'), 'w', encoding='utf-8',
                 newline='') as f:
        w = csv.writer(f)
        w.writerow(['day', 'engine', 'requests', 'pages', 'browser'])
        for (day, name), row in sorted(refer.items()):
            w.writerow([day, name, row['req'], row['pages'], row['browser']])

    with io.open(os.path.join(out_dir, 'traffic.csv'), 'w', encoding='utf-8',
                 newline='') as f:
        w = csv.writer(f)
        w.writerow(['month', 'kind', 'referrer', 'page_views'])
        for (month, kind, host), n in sorted(other.items(),
                                             key=lambda kv: (kv[0][0], -kv[1])):
            w.writerow([month, kind, host, n])

    fmt = [
        '# What the log line actually contains',
        '',
        'Checked rather than assumed, on %d file(s) from %s.' % (len(files), al),
        '',
        '| property | value |',
        '|---|---|',
        '| lines parsed | %d |' % parsed,
        '| lines not matching the combined format | %d |' % unparsed,
        '| lines carrying a user agent | %d |' % have_agent,
        '| lines carrying a referrer | %d |' % have_ref,
        '| requests attributed to a known crawler | %d |'
        % sum(r['req'] for r in crawl.values()),
        '| of those, served with a 200 or 206 | %d |'
        % sum(r['ok'] for r in crawl.values()),
        '| of those, a page rather than an asset | %d |'
        % sum(r['pages'] for r in crawl.values()),
        '| requests not attributed to a crawler | %d |' % human,
        '| arrivals from an answer engine | %d |'
        % sum(r['req'] for r in refer.values()),
        '| distinct page paths the site served to anybody | %d |'
        % len(site_pages),
        '',
        'Requests are counted three ways because they mean three different '
        'things. Every request is what the bandwidth bill sees. A served '
        'request is one the server actually answered, so a redirect chain or a '
        'run of 404s does not count as content taken. A page is a served '
        'request whose path is not a stylesheet, script, font or image, which '
        'is what the argument about content is actually about.',
        '',
        'A referrer count of zero would mean the ratio this paper reports has no',
        'divisor and the paper reports an asymmetry instead. That is a finding',
        'about the log format as much as about the traffic, so it is recorded here',
        'before any analysis is written.',
        '',
    ]
    io.open(os.path.join(out_dir, 'format.md'), 'w', encoding='utf-8').write(
        '\n'.join(fmt))

    print('parsed %d line(s), %d unparsed, %d crawler request(s), %d other'
          % (parsed, unparsed, sum(r['req'] for r in crawl.values()), human))
    print('crawlers.csv: %d row(s)   referrals.csv: %d row(s)'
          % (len(crawl), len(refer)))
    top = sorted(((sum(v['req'] for (d, n2), v in crawl.items() if n2 == n), n)
                  for n in {n for _d, n in crawl}), reverse=True)[:8]
    for n_req, name in top:
        print('   %-22s %7d request(s)' % (name, n_req))
    if refer:
        print('  referrals:')
        for (day, name), row in sorted(refer.items())[:8]:
            print('   %-22s %s  %d' % (name, day, row['req']))
    else:
        print('  no answer-engine referrals found in this window')
    return 0


if __name__ == '__main__':
    raise SystemExit(main())