# -*- coding: utf-8 -*-
"""One polite homepage request per Greek domain, for a technical audit.

Sample frame: Tranco top-1M list PYG5J (2026-08-06), filtered to .gr.
One GET per domain, identifying user agent, 20s timeout, no retries, no crawling
beyond the homepage. Records response metadata and the signals needed to decide
whether a site is an e-shop and how it is built.
"""
import csv
import io
import json
import os
import re
import socket
import ssl
import sys
import time
from concurrent.futures import ThreadPoolExecutor

import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(HERE, 'gr_raw2.jsonl')
N = int(sys.argv[1]) if len(sys.argv) > 1 else 700

UA = ('Mozilla/5.0 (compatible; broikos-research/1.0; +https://broikos.gr/research/; '
      'one homepage request per domain for a published technical audit)')

CTX = ssl.create_default_context()
CTX.check_hostname = False
CTX.verify_mode = ssl.CERT_NONE          # measure TLS separately; do not fail the fetch

# Signatures must be specific. An earlier version used `mage-`, which matches
# `image-` and mislabelled 64% of the sample as Magento.
SHOP = [
    (r'woocommerce|wc-ajax|wp-content/plugins/woocommerce', 'WooCommerce'),
    (r'cdn\.shopify\.com|shopify\.com/s/files|myshopify', 'Shopify'),
    (r'magento|mage/requirejs|magento_|/static/version\d', 'Magento'),
    (r'prestashop', 'PrestaShop'),
    (r'opencart|route=common', 'OpenCart'),
    (r'cs-cart', 'CS-Cart'),
    (r'wixstatic|_partials/wix', 'Wix'),
    (r'bigcommerce', 'BigCommerce'),
    (r'squarespace', 'Squarespace'),
]
CMS = [(r'wp-content|wp-includes', 'WordPress'), (r'drupal', 'Drupal'),
       (r'joomla', 'Joomla'), (r'/_next/', 'Next.js'), (r'nuxt', 'Nuxt')]


def tls_info(host):
    try:
        c = ssl.create_default_context()
        with socket.create_connection((host, 443), timeout=10) as s:
            with c.wrap_socket(s, server_hostname=host) as ss:
                cert = ss.getpeercert()
                return ss.version(), cert.get('notAfter', ''), True
    except ssl.SSLError as e:
        return None, str(e)[:80], False
    except Exception:
        return None, '', False


def fetch(row):
    rank, domain = row
    rec = {'rank': rank, 'domain': domain}
    url = 'https://' + domain
    t0 = time.time()
    try:
        req = urllib.request.Request(url, headers={'User-Agent': UA,
                                                   'Accept': 'text/html,*/*',
                                                   'Accept-Encoding': 'gzip, deflate'})
        r = urllib.request.urlopen(req, timeout=20, context=CTX)
        raw = r.read(400_000)
        rec['status'] = r.status
        rec['elapsed_ms'] = int((time.time() - t0) * 1000)
        rec['final_url'] = r.geturl()
        h = {k.lower(): v for k, v in r.headers.items()}
        rec['headers'] = {k: h.get(k, '') for k in
                          ('server', 'content-encoding', 'strict-transport-security',
                           'content-security-policy', 'x-frame-options',
                           'x-content-type-options', 'referrer-policy',
                           'permissions-policy', 'set-cookie', 'content-type')}
        rec['bytes'] = len(raw)
        enc = h.get('content-encoding', '')
        if 'gzip' in enc:
            import gzip
            try:
                raw = gzip.decompress(raw)
            except Exception:
                pass
        elif 'deflate' in enc:
            import zlib
            try:
                raw = zlib.decompress(raw)
            except Exception:
                pass
        html = raw.decode('utf-8', 'replace')
        rec['html_bytes'] = len(html)
        low = html.lower()
        rec['platform'] = next((n for p, n in SHOP if re.search(p, low)), '')
        rec['platform_all'] = [n for p, n in SHOP if re.search(p, low)]
        rec['cms_all'] = [n for p, n in CMS if re.search(p, low)]
        rec['sig'] = {k: bool(re.search(v, low)) for k, v in {
            'woocommerce': r'woocommerce', 'shopify': r'myshopify|cdn\.shopify',
            'magento_word': r'magento', 'static_version': r'/static/version\d',
            'mage_dash': r'mage-', 'image_dash': r'image-',
            'prestashop': r'prestashop', 'opencart': r'opencart',
        }.items()}
        rec['cms'] = next((n for p, n in CMS if re.search(p, low)), '')
        rec['jsonld_product'] = bool(re.search(r'"@type"\s*:\s*"(product|offer)"', low))
        rec['cart_signal'] = bool(re.search(r'καλάθι|add[-_ ]?to[-_ ]?cart|/cart|/basket|'
                                            r'checkout|προϊόν|αγορά', low))
        rec['viewport'] = bool(re.search(r'<meta[^>]+name=["\']viewport', low))
        rec['lang_attr'] = bool(re.search(r'<html[^>]+lang=', low))
        rec['imgs'] = len(re.findall(r'<img\b', low))
        rec['imgs_alt'] = len(re.findall(r'<img\b[^>]*\balt=', low))
        rec['inputs'] = len(re.findall(r'<input\b', low))
        rec['labels'] = len(re.findall(r'<label\b', low))
        rec['h1'] = len(re.findall(r'<h1\b', low))
        rec['scripts'] = len(re.findall(r'<script\b', low))
        rec['title'] = (re.search(r'<title[^>]*>(.*?)</title>', html, re.S | re.I).group(1).strip()[:120]
                        if re.search(r'<title[^>]*>(.*?)</title>', html, re.S | re.I) else '')
        rec['consent'] = bool(re.search(r'cookiebot|onetrust|cookieconsent|cookie-consent|'
                                        r'gdpr|αποδοχή|συγκατάθεση', low))
    except Exception as e:
        rec['status'] = None
        rec['error'] = '%s: %s' % (type(e).__name__, str(e)[:90])
        rec['elapsed_ms'] = int((time.time() - t0) * 1000)
    v, exp, ok = tls_info(domain)
    rec['tls_version'] = v
    rec['tls_valid'] = ok
    rec['tls_note'] = exp[:80] if not ok else ''
    return rec


rows = []
with io.open(os.path.join(HERE, 'gr_domains.csv'), encoding='utf-8') as f:
    for r in csv.DictReader(f):
        rows.append((int(r['rank']), r['domain']))
rows = rows[:N]
print('fetching %d domains' % len(rows), flush=True)

done = 0
with io.open(OUT, 'w', encoding='utf-8') as out:
    with ThreadPoolExecutor(max_workers=10) as ex:
        for rec in ex.map(fetch, rows):
            out.write(json.dumps(rec, ensure_ascii=False) + '\n')
            done += 1
            if done % 50 == 0:
                print('  %d/%d' % (done, len(rows)), flush=True)
print('wrote %s' % OUT)