#!/usr/bin/env python3
"""Scores one SEO cohort run into src/data/seo-bench/<date>.json (tables the pages render)
and copies sanitised per-call records to public/evidence/seo-runs/<date>/. Deterministic.
Usage: scripts/score-seo-cohort.py evidence/fixtures/seo-cohort-v1.json evidence/seo-runs/YYYY-MM-DD"""
import json, sys, os, glob, math, statistics, shutil
from urllib.parse import urlparse
cohort = json.load(open(sys.argv[1])); d = sys.argv[2]; date = os.path.basename(d.rstrip('/'))
Q, D = cohort['queries'], cohort['domains']
LIST_PRICE = {  # USD per call, catalog list prices read 2026-09-15; treg-routed uses the observed charge
    ('kw','dataforseo'): 0.09, ('kw','serpstat'): 0.0005*len(Q), ('kw','seranking'): 0.00179*len(Q), ('kw','google-ads'): 0.0,
    ('serp','cloro'): 0.0028, ('serp','dataforseo'): 0.002, ('serp','serpstat'): 0.0005, ('serp','serpapi'): 0.015, ('serp','treg-routed'): None,
    ('bl','serpstat'): 0.0025, ('bl','seranking'): 0.0179, ('bl','moz'): 0.0133,
}
recs = [json.load(open(f)) for f in sorted(glob.glob(os.path.join(d, '*.json')))]
def norm(u):
    if not u: return None
    p = urlparse(u); host = p.netloc.lower().removeprefix('www.'); path = p.path.rstrip('/') or '/'
    return host + path
def no_data(r):
    b = r['response']
    if isinstance(b, dict) and isinstance(b.get('error'), dict) and 'not found' in str(b['error'].get('message', '')).lower(): return True
    return False
def ok(r):
    b = r['response']
    if not isinstance(b, dict) and not isinstance(b, list): return False
    if isinstance(b, dict) and ('detail' in b or 'error' in b or '_nonjson' in b): return False
    if isinstance(b, dict) and b.get('status_code') not in (None, 20000): return False
    if isinstance(b, dict) and b.get('success') is False: return False
    return r['_run']['exit'] == 0

# ---------- keyword volume ----------
def kw_map(r):
    p = r['_run']['provider']; b = r['response']; m = {}
    try:
        if p == 'dataforseo':
            for it in b['tasks'][0]['result']: m[it['keyword']] = it.get('search_volume')
        elif p == 'serpstat':
            for it in b['result']['data']: m[it['keyword']] = it.get('region_queries_count')
        elif p == 'seranking':
            for it in b: m[it['keyword']] = it.get('volume') if it.get('is_data_found') else None
        elif p == 'google-ads':
            for it in b['results']:
                km = it.get('keywordMetrics') or {}
                v = km.get('avgMonthlySearches')
                if v is None and km.get('monthlySearchVolumes'):
                    xs = [int(x['monthlySearches']) for x in km['monthlySearchVolumes'] if x.get('monthlySearches') is not None]
                    v = round(sum(xs)/len(xs)) if xs else None
                m[it['text']] = int(v) if v is not None else None
    except Exception as e:
        m['_error'] = str(e)
    return m
kw = {r['_run']['provider']: (r, kw_map(r)) for r in recs if r['_run']['task'] == 'kw'}
ref = kw.get('google-ads', (None, {}))[1]
kw_rows = []
for p, (r, m) in kw.items():
    present = [q for q in Q if q in m]
    nonnull = [q for q in present if m[q] is not None]
    nonzero = [q for q in nonnull if m[q] not in (0, '0')]
    zeros = [q for q in nonnull if m[q] in (0, '0')]
    # agreement with Google Ads where both have a value > 0
    ratios = [math.log10(m[q]/ref[q]) for q in nonzero if ref.get(q) not in (None, 0) and m[q] > 0] if p != 'google-ads' else []
    within2x = sum(1 for x in ratios if abs(x) <= math.log10(2))
    kw_rows.append({'provider': p, 'ok': ok(r), 'latency_ms': r['_run']['latency_ms'], 'cost_usd': LIST_PRICE[('kw', p)],
                    'returned': len(present), 'with_value': len(nonnull), 'reported_zero': len(zeros), 'missing_vs_zero': 'distinguished' if (len(present) - len(nonnull)) > 0 or p in ('seranking','dataforseo') else 'not observed',
                    'within_2x_of_google_ads': (f"{within2x}/{len(ratios)}" if ratios else '—'),
                    'median_log10_ratio_vs_google_ads': (round(statistics.median(ratios), 2) if ratios else None)})
kw_detail = [{'query': q, **{p: kw[p][1].get(q, 'absent') for p in kw}} for q in Q]

# ---------- SERP completeness ----------
def serp_urls(r):
    p = r['_run']['provider']; b = r['response']
    try:
        if p == 'cloro': items = [(x['position'], x['link']) for x in b['result']['organicResults']]
        elif p == 'dataforseo': items = [(x['rank_group'], x['url']) for x in b['tasks'][0]['result'][0]['items'] if x.get('type') == 'organic']
        elif p == 'serpstat': items = [(x['position'], x['url']) for x in b['result']['data']['top']]
        elif p == 'serpapi': items = [(x['position'], x['link']) for x in b['organic_results']]
        elif p == 'treg-routed':
            org = [x for x in b['output']['results'] if x.get('type') == 'organic']
            items = [(i+1, x.get('url')) for i, x in enumerate(org)]
        else: items = []
    except Exception: return []
    return [norm(u) for _, u in sorted(items, key=lambda t: t[0]) if u][:10]
serp = {}
for r in recs:
    if r['_run']['task'] == 'serp': serp.setdefault(r['_run']['provider'], {})[r['_run']['q']] = r
REF = 'cloro'
serp_rows = []
for p, byq in serp.items():
    n = len(byq); oks = [r for r in byq.values() if ok(r)]; nodata = [r for r in byq.values() if no_data(r)]
    lat = sorted(r['_run']['latency_ms'] for r in byq.values())
    counts = [len(serp_urls(r)) for r in oks]
    overlap = []; top3 = []
    for q, r in byq.items():
        if p == REF or not ok(r): continue
        ref_r = serp.get(REF, {}).get(q)
        if not ref_r or not ok(ref_r): continue
        a, b_ = serp_urls(ref_r), serp_urls(r)
        if not a: continue
        overlap.append(len(set(a) & set(b_)) / len(a))
        top3.append(1 if a[:1] and b_[:1] and a[0] == b_[0] else 0)
    charged = [r['response'].get('_treg', {}).get('charged_micro', 0)/1e6 for r in oks if p == 'treg-routed']
    served = {}
    for r in oks:
        s = r['response'].get('_treg', {}).get('served_by') if p == 'treg-routed' else None
        if s: served[s.split('.')[0]] = served.get(s.split('.')[0], 0) + 1
    serp_rows.append({'provider': p, 'calls': n, 'ok': len(oks), 'no_data': len(nodata), 'errors': n - len(oks) - len(nodata), 'p50_ms': lat[len(lat)//2] if lat else None, 'p95_ms': lat[max(0, math.ceil(0.95*len(lat))-1)] if lat else None,
                      'median_results': statistics.median(counts) if counts else None, 'full_top10': sum(1 for c in counts if c >= 10),
                      'overlap_with_live_google': (round(statistics.mean(overlap), 2) if overlap else ('reference' if p == REF else None)),
                      'same_first_result': (f"{sum(top3)}/{len(top3)}" if top3 else ('reference' if p == REF else None)),
                      'cost_usd_per_call': (round(statistics.mean(charged), 5) if charged else LIST_PRICE[('serp', p)]), 'served_by': served or None})

# ---------- backlinks ----------
def bl_vals(r):
    p = r['_run']['provider']; b = r['response']
    try:
        if p == 'serpstat': x = b['result']['data']; return {'refdomains': x.get('referring_domains'), 'backlinks': x.get('backlinks'), 'authority': x.get('sersptat_domain_rank')}
        if p == 'seranking': x = b['summary'][0]; return {'refdomains': x.get('refdomains'), 'backlinks': x.get('backlinks'), 'authority': x.get('domain_inlink_rank')}
        if p == 'moz': x = b['results'][0]; return {'refdomains': x.get('root_domains_to_root_domain'), 'backlinks': x.get('pages_to_root_domain'), 'authority': x.get('domain_authority')}
    except Exception: return {}
    return {}
bl = {}
for r in recs:
    if r['_run']['task'] == 'bl': bl.setdefault(r['_run']['provider'], {})[r['_run']['d']] = r
bl_rows = []; bl_detail = []
for dom in D:
    row = {'domain': dom}
    for p in bl:
        r = bl[p].get(dom); v = bl_vals(r) if r and ok(r) else {}
        row[f'{p}_refdomains'] = v.get('refdomains'); row[f'{p}_backlinks'] = v.get('backlinks'); row[f'{p}_authority'] = v.get('authority')
    rd = [row[f'{p}_refdomains'] for p in bl if row.get(f'{p}_refdomains')]
    row['refdomains_spread'] = (round(max(rd)/min(rd), 1) if len(rd) >= 2 and min(rd) > 0 else None)
    bl_detail.append(row)
def spearman(xs, ys):
    n = len(xs);
    if n < 3: return None
    rx = {v: i for i, v in enumerate(sorted(xs))}; ry = {v: i for i, v in enumerate(sorted(ys))}
    dd = sum((rx[a]-ry[b])**2 for a, b in zip(xs, ys)); return round(1 - 6*dd/(n*(n*n-1)), 2)
for p, byd in bl.items():
    oks = [r for r in byd.values() if ok(r)]; lat = sorted(r['_run']['latency_ms'] for r in byd.values())
    xs, ys = [], []
    for row in bl_detail:
        a, b_ = row.get(f'{p}_refdomains'), row.get('serpstat_refdomains') if p != 'serpstat' else row.get('seranking_refdomains')
        if a and b_: xs.append(a); ys.append(b_)
    bl_rows.append({'provider': p, 'calls': len(byd), 'ok': len(oks), 'p50_ms': lat[len(lat)//2] if lat else None, 'p95_ms': lat[max(0, math.ceil(0.95*len(lat))-1)] if lat else None,
                    'cost_usd_per_call': LIST_PRICE[('bl', p)], 'rank_corr_refdomains_vs_other': spearman(xs, ys)})
spreads = [r['refdomains_spread'] for r in bl_detail if r['refdomains_spread']]
summary = {'date': date, 'cohort': cohort['version'], 'queries': len(Q), 'domains': len(D), 'calls': len(recs), 'ok': sum(ok(r) for r in recs),
           'est_cost_usd': round(sum((LIST_PRICE.get((r['_run']['task'], r['_run']['provider'])) or (r['response'].get('_treg', {}).get('charged_micro', 0)/1e6 if isinstance(r['response'], dict) else 0)) for r in recs), 3),
           'keyword': kw_rows, 'keyword_detail': kw_detail, 'serp': serp_rows, 'backlinks': bl_rows, 'backlinks_detail': bl_detail,
           'backlinks_median_spread': (statistics.median(spreads) if spreads else None), 'backlinks_max_spread': (max(spreads) if spreads else None)}
os.makedirs('src/data/seo-bench', exist_ok=True); json.dump(summary, open(f'src/data/seo-bench/{date}.json', 'w'), indent=1)
pub = f'public/evidence/seo-runs/{date}'; os.makedirs(pub, exist_ok=True)
for r in recs:  # publish per-call records; SERP/backlink data is public data about public sites, keyword volumes are provider outputs
    json.dump(r, open(os.path.join(pub, r['_run']['label'] + '.json'), 'w'))
import csv as _csv
tdir = os.path.join(pub, 'tables'); os.makedirs(tdir, exist_ok=True)
for name in ('keyword','keyword_detail','serp','backlinks','backlinks_detail'):
    rows_ = summary[name]
    if not rows_: continue
    keys = [k for k in rows_[0].keys() if k != 'rows']
    with open(os.path.join(tdir, f'{name}.csv'), 'w', newline='') as fh:
        w = _csv.DictWriter(fh, fieldnames=keys, extrasaction='ignore'); w.writeheader()
        for r in rows_: w.writerow({k: (json.dumps(v) if isinstance(v, (dict, list)) else v) for k, v in r.items() if k in keys})
summary['files'] = {'tables': sorted(os.listdir(tdir)), 'raw_count': len(recs)}
json.dump(summary, open(f'src/data/seo-bench/{date}.json', 'w'), indent=1)
print(json.dumps({k: summary[k] for k in ('calls','ok','est_cost_usd','backlinks_median_spread')}))
for row in kw_rows: print('KW ', row)
for row in serp_rows: print('SERP', row)
for row in bl_rows: print('BL ', row)