# -*- coding: utf-8 -*-
"""Pull archived access logs for one site into a local, unpublished store.

    python pull_logs.py broikos.gr              every archive it holds
    python pull_logs.py broikos.gr --months 3   the three most recent

Nothing here is published and nothing is written back to the server. The files
carry visitor IP addresses, which are personal data, so they land in `raw/`, which
is listed in the paper's never-publish set beside the merchant key from paper 02.
What the paper ships is the aggregate tables and the code that produced them.

The default site is the author's own. Client sites are pulled only when their
owner has agreed, which is a decision recorded in the plan rather than a flag on
this script.
"""
import argparse
import ftplib
import io
import os
import re
import sys

HTDOCS = r'C:\xampp\htdocs'
HERE = os.path.dirname(os.path.abspath(__file__))
RAW = os.path.join(HERE, 'raw')
LOG_DIRS = ('/logs', '/access-logs')

sys.path.insert(0, HERE)
from log_inventory import creds, listing                           # noqa: E402


def pull(site, months, dry):
    c = creds(site)
    if not c:
        print('%s: no usable credential' % site)
        return 1
    host, user, pwd = c
    ftp = ftplib.FTP_TLS(timeout=60)
    ftp.connect(host, 21)
    ftp.login(user, pwd)
    ftp.prot_p()

    dest = os.path.join(RAW, site)
    os.makedirs(dest, exist_ok=True)

    wanted = []
    for d in LOG_DIRS:
        for name, size, when in listing(ftp, d):
            if not name.endswith('.gz'):
                continue
            if 'ftp_log' in name:
                continue                     # file transfers, not visitors
            wanted.append((d + '/' + name, name, size))
    wanted.sort(key=lambda x: x[2], reverse=True)
    if months:
        wanted = wanted[:months]

    total = 0
    for path, name, size in wanted:
        local = os.path.join(dest, name)
        if os.path.exists(local) and os.path.getsize(local) == size:
            print('  have   %-46s %8.1f KB' % (name, size / 1024))
            continue
        if dry:
            print('  would  %-46s %8.1f KB' % (name, size / 1024))
            continue
        with io.open(local, 'wb') as f:
            ftp.retrbinary('RETR %s' % path, f.write)
        got = os.path.getsize(local)
        total += got
        print('  pulled %-46s %8.1f KB' % (name, got / 1024))
    ftp.quit()
    print('\n%s: %d archive(s) considered, %.1f MB fetched into %s'
          % (site, len(wanted), total / 1e6, dest))
    print('raw logs are never published: aggregates and code only')
    return 0


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('site', nargs='?', default='broikos.gr')
    ap.add_argument('--months', type=int, default=0,
                    help='keep only the N largest archives, 0 for all')
    ap.add_argument('--dry-run', action='store_true')
    a = ap.parse_args()
    return pull(a.site, a.months, a.dry_run)


if __name__ == '__main__':
    raise SystemExit(main())