#!/usr/bin/env python3
"""Scores one audit-cost run (scripts/run-audit-cost.py): where the audited domain ranks for the
cohort's queries on each route, volumes returned, ranked-keyword and backlink totals, the on-page
crawl summary, and what each route's audit cost at list price and on the Treg ledger. No provider
call. Writes summary.json and tables/*.csv (no result URLs other than the audited domain's own).
Usage: scripts/score-audit-cost.py evidence/fixtures/seo-cohort-v1.json evidence/audit-cost/YYYY-MM-DD"""
import json, sys, os, csv, glob, statistics as st
from collections import defaultdict
cohort = json.load(open(sys.argv[1])); out = sys.argv[2]; raw = os.path.join(out, 'raw'); tdir = os.path.join(out, 'tables'); os.makedirs(tdir, exist_ok=True)
Q = cohort['queries']; led = json.load(open(os.path.join(out, 'ledger.json'))); domain = led['domain']
def load(label):
    f = os.path.join(raw, label + '.json')
    return json.load(open(f)) if os.path.exists(f) else None
def norm_dom(d): return (d or '').lower().removeprefix('www.')
def is_target(d): d = norm_dom(d); return d == domain or d.endswith('.' + domain)
# ---- rankings for the 50 queries
rows = []; serp = {'dataforseo': {}, 'serpstat': {}}; serp_meta = defaultdict(lambda: {'calls': 0, 'ok': 0, 'no_data': 0, 'errors': 0, 'lat': []})
for i, q in enumerate(Q):
    k = f'q{i+1:02d}'; row = {'query': q}
    r = load(f'serp-dataforseo-{k}'); m = serp_meta['dataforseo']; m['calls'] += 1
    pos = None; got = False
    if r:
        m['lat'].append(r['_run']['latency_ms'])
        try:
            items = r['response']['tasks'][0]['result'][0]['items'] or []; got = True
            orgs = [it for it in items if it.get('type') == 'organic']
            hit = next((it for it in orgs if is_target(it.get('domain'))), None)
            pos = hit['rank_group'] if hit else None
            row['dataforseo_organic_rows'] = len(orgs); row['dataforseo_ai_overview'] = any(it.get('type') == 'ai_overview' for it in items)
        except Exception: pass
    m['ok' if got else 'errors'] += 1; row['dataforseo_position'] = pos if got else 'error'; serp['dataforseo'][q] = pos
    r = load(f'serp-serpstat-{k}'); m = serp_meta['serpstat']; m['calls'] += 1
    pos = None; got = False; nd = False
    if r:
        m['lat'].append(r['_run']['latency_ms'])
        try:
            if (r['response'].get('error') or {}).get('code') in (32017, -32000): got = True; nd = True; row['serpstat_rows'] = 0  # Serpstat's 'Keyword not found' / 'Data not found': the query is not in its stored index
            else:
                top = r['response']['result']['data'].get('top') or []; got = True; nd = len(top) == 0
                hit = next((it for it in top if is_target(it.get('domain'))), None); pos = hit['position'] if hit else None
                row['serpstat_rows'] = len(top)
        except Exception: pass
    if got and nd: m['no_data'] += 1
    elif got: m['ok'] += 1
    else: m['errors'] += 1
    row['serpstat_position'] = 'no data' if nd else (pos if got else 'error'); serp['serpstat'][q] = None if nd else pos
    rows.append(row)
with open(os.path.join(tdir, 'positions.csv'), 'w', newline='') as f:
    w = csv.DictWriter(f, fieldnames=['query', 'dataforseo_position', 'dataforseo_organic_rows', 'dataforseo_ai_overview', 'serpstat_position', 'serpstat_rows']); w.writeheader(); [w.writerow(r) for r in rows]
# ---- keyword volumes
kw = {}
r = load('kw-dataforseo-all')
if r:
    try: kw['dataforseo'] = {it['keyword']: it.get('search_volume') for it in r['response']['tasks'][0]['result']}
    except Exception: kw['dataforseo'] = {}
r = load('kw-serpstat-all')
if r:
    try: kw['serpstat'] = {it['keyword']: it.get('region_queries_count') for it in r['response']['result']['data']}
    except Exception: kw['serpstat'] = {}
with open(os.path.join(tdir, 'volumes.csv'), 'w', newline='') as f:
    w = csv.writer(f); w.writerow(['query', 'dataforseo_volume', 'serpstat_volume'])
    for q in Q: w.writerow([q, kw.get('dataforseo', {}).get(q, ''), kw.get('serpstat', {}).get(q, '')])
# ---- ranked keywords
rk = {}
r = load('rk-dataforseo')
if r:
    try:
        res = r['response']['tasks'][0]['result'][0]; items = res.get('items') or []
        rk['dataforseo'] = {'total_count': res.get('total_count'), 'rows_returned': len(items), 'provider_cost': r['response'].get('cost'),
                            'top3': sum(1 for it in items if (it['ranked_serp_element']['serp_item'].get('rank_group') or 999) <= 3),
                            'top10': sum(1 for it in items if (it['ranked_serp_element']['serp_item'].get('rank_group') or 999) <= 10),
                            'cohort_queries_in_rows': sum(1 for it in items if it['keyword_data']['keyword'] in Q)}
    except Exception as e: rk['dataforseo'] = {'error': str(e)}
r = load('rk-serpstat')
if r:
    try:
        res = r['response']['result']; items = res.get('data') or []
        rk['serpstat'] = {'total_count': (res.get('summary_info') or {}).get('total'), 'rows_returned': len(items), 'left_lines': (res.get('summary_info') or {}).get('left_lines'),
                          'top3': sum(1 for it in items if (it.get('position') or 999) <= 3), 'top10': sum(1 for it in items if (it.get('position') or 999) <= 10),
                          'cohort_queries_in_rows': sum(1 for it in items if it.get('keyword') in Q)}
    except Exception as e: rk['serpstat'] = {'error': str(e)}
r = load('rkq-serpstat')
if r:
    try:
        items = r['response']['result'].get('data') or []
        rk['serpstat_targeted'] = {'rows_returned': len(items), 'positions': {it['keyword']: it.get('position') for it in items}}
    except Exception as e: rk['serpstat_targeted'] = {'error': str(e), 'response': r['response'].get('error')}
# ---- backlinks
bl = {}
r = load('bl-dataforseo')
if r:
    try:
        res = r['response']['tasks'][0]['result'][0]
        bl['dataforseo'] = {k: res.get(k) for k in ['backlinks', 'referring_domains', 'referring_main_domains', 'referring_pages', 'referring_ips', 'rank', 'broken_backlinks', 'first_seen', 'lost_date', 'crawled_pages']}
    except Exception as e: bl['dataforseo'] = {'error': str(e)}
r = load('bl-serpstat')
if r:
    try:
        d = r['response']['result']['data']; bl['serpstat'] = {k: d.get(k) for k in d if isinstance(d.get(k), (int, float, str))}
    except Exception as e: bl['serpstat'] = {'error': str(e)}
r = load('rd-dataforseo')
if r:
    try:
        res = r['response']['tasks'][0]['result'][0]; bl['dataforseo_referring_domains_list'] = {'total_count': res.get('total_count'), 'rows_returned': len(res.get('items') or [])}
    except Exception as e: bl['dataforseo_referring_domains_list'] = {'error': str(e)}
ov = None
r = load('ov-serpstat')
if r:
    try: ov = r['response']['result']['data'][0]
    except Exception as e: ov = {'error': str(e)}
# ---- on-page
onpage = None
r = load('onpage-dataforseo-summary')
if r:
    try:
        res = r['response']['tasks'][0]['result'][0]; di = res.get('domain_info') or {}; pm = res.get('page_metrics') or {}
        checks = pm.get('checks') or {}
        onpage = {'crawl_progress': res.get('crawl_progress'), 'crawl_status': res.get('crawl_status'), 'total_pages': di.get('total_pages'), 'pages_in_queue': (res.get('crawl_status') or {}).get('pages_in_queue'),
                  'onpage_score': pm.get('onpage_score'), 'links_external': pm.get('links_external'), 'links_internal': pm.get('links_internal'), 'broken_links': pm.get('broken_links'), 'duplicate_title': pm.get('duplicate_title'), 'duplicate_description': pm.get('duplicate_description'),
                  'checks_nonzero': {k: v for k, v in checks.items() if v}, 'checks_total': len(checks), 'checks_failed_kinds': sum(1 for v in checks.values() if v)}
    except Exception as e: onpage = {'error': str(e)}
# ---- Serpstat credits per call, from the left_lines counter it returns on every response
ss = []
for f in glob.glob(os.path.join(raw, '*serpstat*.json')):
    r = json.load(open(f)); ll = None
    try: ll = r['response']['result']['summary_info']['left_lines']
    except Exception:
        try: ll = r['response']['result']['data'].get('summary_info', {}).get('left_lines')
        except Exception: pass
    ss.append((r['_run']['ts'], r['_run']['label'], r['_run']['task'], ll))
ss.sort(); serpstat_credits = []
for (ts, label, task, ll), (pts, plabel, ptask, pll) in zip(ss[1:], ss[:-1]):
    serpstat_credits.append({'label': label, 'task': task, 'credits': (pll - ll) if (ll is not None and pll is not None) else None})
with open(os.path.join(tdir, 'serpstat-credits.csv'), 'w', newline='') as f:
    w = csv.DictWriter(f, fieldnames=['label', 'task', 'credits']); w.writeheader(); [w.writerow(r) for r in serpstat_credits]
credits_by_task = defaultdict(list)
for r in serpstat_credits:
    if r['credits'] is not None: credits_by_task[r['task']].append(r['credits'])
# ---- cost: list price per label and ledger per endpoint
by_call = defaultdict(list)
for e in led['entries']: by_call[e['call_id']].append(e)
ledger_ep = defaultdict(lambda: {'calls': 0, 'settled_micro': 0})
for cid, es in by_call.items():
    kinds = {e['kind'] for e in es}; ep = es[0]['endpoint_id']
    if 'settle' in kinds: c = sum(e['meta'].get('settled_micro', -e['amount_micro']) for e in es if e['kind'] == 'settle')
    elif 'release' in kinds: c = 0
    else: c = -sum(e['amount_micro'] for e in es if e['kind'] == 'reserve')
    ledger_ep[ep]['calls'] += 1; ledger_ep[ep]['settled_micro'] += c
prov_of = {'dataforseo.x.serp-google-organic-live-advanced': 'dataforseo', 'dataforseo.google.keywords.volume': 'dataforseo', 'dataforseo.google.domain.ranked_keywords': 'dataforseo', 'dataforseo.web.backlinks.summary': 'dataforseo', 'dataforseo.web.linking_domains.list': 'dataforseo', 'dataforseo.x.on-page-task-post': 'dataforseo', 'dataforseo.x.on-page-summary-id': 'dataforseo',
           'serpstat.google.serp.organic': 'serpstat', 'serpstat.google.keywords.volume': 'serpstat', 'serpstat.google.domain.ranked_keywords': 'serpstat', 'serpstat.web.backlinks.summary': 'serpstat', 'serpstat.google.domain.overview': 'serpstat'}
step_of = {'serp-google-organic-live-advanced': 'rankings, 50 queries', 'serp.organic': 'rankings, 50 queries', 'keywords.volume': 'volumes, 50 keywords', 'domain.ranked_keywords': 'ranked keywords', 'backlinks.summary': 'backlink summary', 'linking_domains.list': 'referring domains, 100', 'on-page-task-post': 'on-page crawl, 100 pages', 'on-page-summary-id': 'on-page crawl, 100 pages', 'domain.overview': 'domain overview'}
def step(ep): return next((v for k, v in step_of.items() if ep.endswith(k) or k in ep), ep)
# provider-reported cost (DataForSEO puts it in the response)
prov_cost = defaultdict(float)
for f in glob.glob(os.path.join(raw, '*.json')):
    r = json.load(open(f)); c = r['response'].get('cost') if isinstance(r['response'], dict) else None
    if isinstance(c, (int, float)): prov_cost[r['_run']['task']] += c
cost_rows = []
for ep, v in sorted(ledger_ep.items()):
    if ep not in prov_of: continue
    cost_rows.append({'route': prov_of[ep], 'endpoint': ep, 'step': step(ep), 'calls': v['calls'], 'ledger_usd': round(v['settled_micro'] / 1e6, 4)})
with open(os.path.join(tdir, 'cost.csv'), 'w', newline='') as f:
    w = csv.DictWriter(f, fieldnames=['route', 'endpoint', 'step', 'calls', 'ledger_usd']); w.writeheader(); [w.writerow(r) for r in cost_rows]
totals = defaultdict(float)
for r in cost_rows: totals[r['route']] += r['ledger_usd']
unmatched = {ep: v for ep, v in ledger_ep.items() if ep not in prov_of}
summary = {'date': out.rstrip('/').split('/')[-1], 'domain': domain, 'window': {'started': led['started'], 'ended': led['ended']}, 'balance_before': led['balance_before'], 'balance_after': led['balance_after'],
           'serp_routes': {p: {**{k: v for k, v in m.items() if k != 'lat'}, 'p50_ms': int(st.median(m['lat'])) if m['lat'] else None, 'p95_ms': int(sorted(m['lat'])[int(0.95 * (len(m['lat']) - 1))]) if m['lat'] else None,
                               'ranked_for': sum(1 for q in Q if serp[p].get(q)), 'top10': sum(1 for q in Q if serp[p].get(q) and serp[p][q] <= 10)} for p, m in serp_meta.items()},
           'positions_agree': sum(1 for q in Q if serp['dataforseo'].get(q) is not None and serp['serpstat'].get(q) is not None and serp['dataforseo'][q] == serp['serpstat'][q]),
           'positions_both': sum(1 for q in Q if serp['dataforseo'].get(q) is not None and serp['serpstat'].get(q) is not None),
           'volumes': {p: {'returned': sum(1 for q in Q if kw[p].get(q) is not None), 'zero': sum(1 for q in Q if kw[p].get(q) == 0)} for p in kw},
           'volumes_equal_on_shared': sum(1 for q in Q if kw.get('dataforseo', {}).get(q) is not None and kw.get('serpstat', {}).get(q) is not None and kw['dataforseo'][q] == kw['serpstat'][q]),
           'volumes_shared': sum(1 for q in Q if kw.get('dataforseo', {}).get(q) is not None and kw.get('serpstat', {}).get(q) is not None),
           'serpstat_credits_by_task': {k: {'calls_measured': len(v), 'credits_total': sum(v), 'credits_min': min(v), 'credits_max': max(v)} for k, v in credits_by_task.items()}, 'ranked_keywords': rk, 'backlinks': bl, 'serpstat_overview': ov, 'onpage': onpage,
           'cost': {'by_endpoint': cost_rows, 'total_by_route_usd': {k: round(v, 4) for k, v in totals.items()}, 'provider_reported_by_task_usd': {k: round(v, 4) for k, v in prov_cost.items()}, 'ledger_entries': len(led['entries']), 'unmatched_endpoints': unmatched,
                    'balance_delta_usd': round(led['balance_before'] - led['balance_after'], 4) if led['balance_before'] and led['balance_after'] else None}}
json.dump(summary, open(os.path.join(out, 'summary.json'), 'w'), indent=1)
print(json.dumps(summary, indent=1))