#!/usr/bin/env python3
"""Scores scripts/run-ai-answer-repeats.py with no provider call. For each route and prompt: brands named (fixed list),
cited domains and cited URLs per run, then (a) how much of the ten-run list k runs see, (b) how far two disjoint k-run
panels sit apart on one brand's mention rate, every split enumerated, (c) identical answers, sources per call, first-named
brand, and settled cost from the ledger. Published copies hold counts, brand names and domains; answers and URLs stay in
raw/. A domain that matches the exclusion list (scripts/check-exclusions.mjs) is left out of the published rows and counted.
Usage: scripts/score-ai-answer-repeats.py <fixture> <run-dir> <published-out-dir>"""
import json, sys, os, re, glob, itertools, csv, random, statistics as st
from collections import Counter, defaultdict
from urllib.parse import urlsplit, parse_qsl, urlencode
fx = json.load(open(sys.argv[1])); d = sys.argv[2]; out = sys.argv[3]; os.makedirs(out, exist_ok=True)
here = os.path.dirname(os.path.abspath(__file__))
EXCLUDED = [t.lower() for t in re.findall(r"'([^']+)'", re.search(r'EXCLUDED = \[(.*?)\]', open(os.path.join(here, 'check-exclusions.mjs')).read()).group(1))]
BRANDS = ['DataForSEO', 'Ahrefs', 'Semrush', 'SerpApi', 'Serper', 'Moz', 'SE Ranking', 'Serpstat', 'Similarweb', 'Search Console', 'Keywords Everywhere', 'Bright Data', 'ScrapingBee', 'Oxylabs', 'Zenserp', 'ValueSERP', 'Apify', 'ScrapeCreators', 'SearchAPI', 'Ubersuggest', 'Mangools', 'Majestic', 'Treg', 'Cloro', 'Nimble', 'Exa', 'Tavily',
          'Apollo', 'Clearbit', 'People Data Labs', 'Hunter', 'ZoomInfo', 'Lusha', 'Clay', 'Crunchbase', 'BuiltWith', 'Coresignal', 'Crustdata', 'Cognism', 'Prospeo', 'Findymail', 'FullEnrich', 'BetterContact', 'Abstract API', 'Wappalyzer', 'Datablist', 'Snov', 'Dropcontact', 'Icypeas', 'ZeroBounce', 'NeverBounce', 'Explorium', 'Wiza', 'MillionVerifier', 'Bouncer', 'Anymail',
          'Meta Ad Library', 'Ads Transparency', 'Foreplay', 'AdSpy', 'BigSpy', 'Panoramata', 'Adyntel', 'Minea', 'Swipe Files']
BRANDS = list(dict.fromkeys(BRANDS)); PATS = {b: re.compile(r'(?<![\w-])' + re.escape(b.lower()) + r'(?![\w-])') for b in BRANDS}
def clean(u):
    p = urlsplit(u or ''); q = [(k, v) for k, v in parse_qsl(p.query) if not k.lower().startswith('utm_') and k.lower() not in ('ref', 'srsltid', 'gclid', 'fbclid')]
    return p.netloc.lower().removeprefix('www.'), p.netloc.lower().removeprefix('www.') + p.path.rstrip('/') + ('?' + urlencode(q) if q else '')
def norm(f):
    rec = json.load(open(f)); m = rec['_run']; r = rec['response']; text, urls = '', []
    if m['route'] == 'perplexity-api':
        results = {}; 
        for o in r.get('output') or []:
            if o.get('type') == 'search_results': results.update({x['id']: x['url'] for x in o['results']})
            if o.get('type') == 'message': text += ''.join(c.get('text') or '' for c in o['content'])
        urls = [results[int(n)] for n in dict.fromkeys(re.findall(r'\[(\d+)\]', text)) if int(n) in results]
        cost = ((r.get('usage') or {}).get('cost') or {}).get('total_cost')
    else:
        res = r.get('result') or {}; text = res.get('text') or ''; urls = [s.get('url') for s in res.get('sources') or [] if s.get('url')]; cost = None
    low = text.lower(); hits = {b: p.search(low).start() for b, p in PATS.items() if p.search(low)}
    pairs = [clean(u) for u in urls]
    return {'route': m['route'], 'id': m['id'], 'run': m['run'], 'latency_ms': m['latency_ms'], 'ok': bool(text.strip()), 'text': text, 'brands': set(hits), 'first': min(hits, key=hits.get) if hits else None,
            'domains': {a for a, _ in pairs if a}, 'urls': {b for _, b in pairs if b}, 'provider_cost': cost}
calls = [norm(f) for f in sorted(glob.glob(os.path.join(d, 'raw', '*-r[0-9][0-9].json')))]
K = [1, 2, 3, 5, 7, 9]
def coverage(runs, field):
    """share of the full list a k-run sample sees, over every k-subset, and items seen in exactly one run"""
    union = set().union(*[r[field] for r in runs]) if runs else set()
    if not union: return None
    res = {}
    for k in K:
        vals = [len(set().union(*[runs[i][field] for i in c])) / len(union) for c in itertools.combinations(range(len(runs)), k)]
        res[k] = sum(vals) / len(vals)
    cnt = Counter(x for r in runs for x in r[field]); n = len(runs)
    return {'union': len(union), 'coverage': res, 'in_one_run': sum(v == 1 for v in cnt.values()), 'in_every_run': sum(v == n for v in cnt.values()), 'mean_per_run': st.mean(len(r[field]) for r in runs)}
def jaccard(runs, field):
    v = [len(a[field] & b[field]) / len(a[field] | b[field]) for a, b in itertools.combinations(runs, 2) if a[field] | b[field]]
    return st.mean(v) if v else None
def split_gap(runs, k):
    """two disjoint k-run panels: mean absolute gap in mention rate per brand, share of brands 20+ points apart"""
    idx = range(len(runs)); gaps = defaultdict(list); union = set().union(*[r['brands'] for r in runs])
    if not union: return None
    for a in itertools.combinations(idx, k):
        rest = [i for i in idx if i not in a]
        for b in itertools.combinations(rest, k):
            if a > b: continue
            for br in union:
                ra = sum(br in runs[i]['brands'] for i in a) / k; rb = sum(br in runs[i]['brands'] for i in b) / k
                gaps[br].append(abs(ra - rb))
    allg = [g for v in gaps.values() for g in v]
    return {'mean_gap': sum(allg) / len(allg), 'share_20plus': sum(g >= 0.2 - 1e-9 for g in allg) / len(allg), 'share_40plus': sum(g >= 0.4 - 1e-9 for g in allg) / len(allg)}
def score_gap(all_runs, k, draws=2000):
    """a tracker's brand score: mentions over all prompts' runs. Two disjoint k-run panels per prompt, seeded draws;
    mean absolute gap for brands whose all-runs score is at least 10%."""
    rnd = random.Random(20261104); brands = Counter(x for rs in all_runs for r in rs for x in r['brands']); total = sum(len(rs) for rs in all_runs)
    keep = [b for b, n in brands.items() if n / total >= 0.1]; gaps = []
    for _ in range(draws):
        ca, cb = Counter(), Counter(); n = 0
        for rs in all_runs:
            if len(rs) < 2 * k: continue  # a prompt with a failed call sits out the panel size it cannot fill
            pick = rnd.sample(range(len(rs)), 2 * k); n += k
            for i in pick[:k]: ca.update(rs[i]['brands'])
            for i in pick[k:]: cb.update(rs[i]['brands'])
        gaps += [abs(ca[b] - cb[b]) / n for b in keep]
    return {'brands_scored': len(keep), 'mean_gap': round(sum(gaps) / len(gaps), 3), 'share_5plus': round(sum(g >= 0.05 for g in gaps) / len(gaps), 3)}
rows, brand_rows, domain_rows, dropped = [], [], [], 0
routes = sorted({c['route'] for c in calls}); ids = [p['id'] for p in fx['prompts']]
for rt in routes:
    for pid in ids:
        runs = sorted([c for c in calls if c['route'] == rt and c['id'] == pid and c['ok']], key=lambda c: c['run'])
        if len(runs) < 2: continue
        row = {'route': rt, 'id': pid, 'runs_ok': len(runs), 'runs_with_sources': sum(bool(r['urls']) for r in runs), 'identical_answer_pairs': sum(a['text'] == b['text'] for a, b in itertools.combinations(runs, 2))}
        for field in ('brands', 'domains', 'urls'):
            row[field] = coverage(runs, field); row[field + '_jaccard'] = jaccard(runs, field)
        row['split_gap'] = {k: split_gap(runs, k) for k in (1, 2, 3, 5) if len(runs) >= 2 * k}
        firsts = Counter(r['first'] for r in runs if r['first']); row['first_named_modal'] = firsts.most_common(1)[0][1] if firsts else 0
        rows.append(row)
        for b, n in Counter(x for r in runs for x in r['brands']).most_common(): brand_rows.append({'route': rt, 'prompt': pid, 'brand': b, 'runs_named': n, 'runs': len(runs)})
        for dm, n in Counter(x for r in runs for x in r['domains']).most_common():
            if any(t in dm for t in EXCLUDED): dropped += 1; continue
            domain_rows.append({'route': rt, 'prompt': pid, 'domain': dm, 'runs_cited': n, 'runs': len(runs)})
def avg(rs, fn):
    v = [fn(r) for r in rs]; v = [x for x in v if x is not None]; return round(sum(v) / len(v), 3) if v else None
summary = {}
for rt in routes:
    rs = [r for r in rows if r['route'] == rt]; cs = [c for c in calls if c['route'] == rt]; s = {'calls': len(cs), 'ok': sum(c['ok'] for c in cs), 'with_sources': sum(bool(c['urls']) for c in cs), 'median_latency_s': round(st.median(c['latency_ms'] for c in cs) / 1000, 1), 'identical_answer_pairs': sum(r['identical_answer_pairs'] for r in rs)}
    for field in ('brands', 'domains', 'urls'):
        s[field] = {'union_per_prompt': avg(rs, lambda r: r[field] and r[field]['union']), 'mean_per_run': avg(rs, lambda r: r[field] and r[field]['mean_per_run']), 'jaccard': avg(rs, lambda r: r[field + '_jaccard']),
                    'coverage': {k: avg(rs, lambda r: r[field] and r[field]['coverage'][k]) for k in K}, 'in_one_run_share': avg(rs, lambda r: r[field] and r[field]['in_one_run'] / r[field]['union']), 'in_every_run_share': avg(rs, lambda r: r[field] and r[field]['in_every_run'] / r[field]['union'])}
    s['split_gap'] = {k: {m: avg(rs, lambda r: (r['split_gap'].get(k) or {}).get(m)) for m in ('mean_gap', 'share_20plus', 'share_40plus')} for k in (1, 2, 3, 5)}
    s['first_named_modal_of_10'] = avg(rs, lambda r: r['first_named_modal'])
    s['brand_rate_bands'] = dict(Counter(('0-20%' if b['runs_named'] / b['runs'] <= .2 else '30-70%' if .3 <= b['runs_named'] / b['runs'] <= .7 else '80-90%' if b['runs_named'] < b['runs'] else '100%') if True else '' for b in brand_rows if b['route'] == rt))
    by = [sorted([c for c in cs if c['id'] == pid and c['ok']], key=lambda c: c['run']) for pid in ids]
    s['score_gap'] = {k: score_gap(by, k) for k in (1, 3, 5)}
    src = [c for c in cs if c['ok'] and c['urls']]; nos = [c for c in cs if c['ok'] and not c['urls']]
    s['sourced_vs_unsourced'] = {'sourced': len(src), 'unsourced': len(nos), 'brands_per_answer_sourced': round(st.mean(len(c['brands']) for c in src), 2) if src else None, 'brands_per_answer_unsourced': round(st.mean(len(c['brands']) for c in nos), 2) if nos else None,
        'latency_s_sourced': round(st.median(c['latency_ms'] for c in src) / 1000, 1) if src else None, 'latency_s_unsourced': round(st.median(c['latency_ms'] for c in nos) / 1000, 1) if nos else None, 'unsourced_per_prompt': dict(Counter(c['id'] for c in nos))}
    pv = []
    for pid in ids:
        a = [c for c in src if c['id'] == pid]; b = [c for c in nos if c['id'] == pid]
        if a and b:
            ra = Counter(x for c in a for x in c['brands']); rb = Counter(x for c in b for x in c['brands'])
            for br in set(ra) | set(rb): pv.append(abs(ra[br] / len(a) - rb[br] / len(b)))
    s['sourced_vs_unsourced']['mean_rate_gap'] = round(st.mean(pv), 3) if pv else None
    summary[rt] = s
ENDPOINT = {'chatgpt-web': 'cloro.ai-search.chatgpt.scrape', 'aimode': 'cloro.google.serp.ai_mode', 'gemini': 'cloro.ai-search.gemini.scrape', 'perplexity-api': 'perplexity.web.answer.fast'}
ledger = json.load(open(os.path.join(d, 'raw', 'ledger.json'))); settled = defaultdict(dict)
for e in ledger['entries']:
    if e.get('kind') == 'settle' and e.get('created_at', '') >= ledger['started']: settled[e.get('endpoint_id')][e['call_id']] = (e.get('meta') or {}).get('settled_micro', 0)
for rt in routes:
    v = list(settled.get(ENDPOINT[rt], {}).values()); summary[rt]['ledger'] = {'settle_rows': len(v), 'settled_usd': round(sum(v) / 1e6, 4), 'per_call_usd': round(sum(v) / len(v) / 1e6, 5) if v else None}
res = {'run': os.path.basename(d.rstrip('/')), 'fixture': fx['name'], 'calls': len(calls), 'domain_rows_dropped_exclusion': dropped, 'balance_before': ledger.get('balance_before'), 'balance_after': ledger.get('balance_after'), 'summary': summary,
       'per_prompt': [{k: v for k, v in r.items()} for r in rows]}
json.dump(res, open(os.path.join(out, 'summary.json'), 'w'), indent=1)
for name, data, cols in (('brands.csv', brand_rows, ['route', 'prompt', 'brand', 'runs_named', 'runs']), ('domains.csv', domain_rows, ['route', 'prompt', 'domain', 'runs_cited', 'runs'])):
    w = csv.DictWriter(open(os.path.join(out, name), 'w', newline=''), fieldnames=cols); w.writeheader(); w.writerows(data)
print(json.dumps(summary, indent=0))
