#!/usr/bin/env python3
"""Scores an AI-answer check run (scripts/run-ai-answer-check.py) into one published file per run:
per call, what came back that a tracker could store as proof the answer was live (answer text, cited
URLs, the model name, a timestamp, whether web search ran, the search queries), the freshness grade
from grades.json, and the brands named. Answers or sources that name a term on the site's exclusion
list (scripts/check-exclusions.mjs) are dropped from the published copy and counted. No provider call.
Usage: scripts/score-ai-answer-check.py evidence/fixtures/ai-answer-check-v1.json evidence/ai-answer-checks/YYYY-MM-DD"""
import json, sys, os, re, glob
fx = json.load(open(sys.argv[1])); d = sys.argv[2]
grades = json.load(open(os.path.join(d, 'grades.json')))['grades']
here = os.path.dirname(os.path.abspath(__file__))
EXCLUDED = [t.lower() for t in re.findall(r"'([^']+)'", re.search(r'EXCLUDED = \[(.*?)\]', open(os.path.join(here, 'check-exclusions.mjs')).read()).group(1))]
# vendor names counted in the panel answers; a name counts once per answer, case-insensitive
BRANDS = ['DataForSEO', 'Ahrefs', 'Semrush', 'SerpApi', 'Serper', 'Moz', 'SE Ranking', 'Serpstat', 'Similarweb', 'Search Console', 'Keywords Everywhere', 'Bright Data', 'ScrapingBee', 'Oxylabs', 'Zenserp', 'ValueSERP', 'Apify', 'ScrapeCreators', 'SearchAPI', 'Ubersuggest', 'Mangools', 'Majestic', 'Treg',
          'Apollo', 'Clearbit', 'People Data Labs', 'Hunter', 'ZoomInfo', 'Lusha', 'Clay', 'Crunchbase', 'BuiltWith', 'Coresignal', 'Crustdata', 'Cognism', 'Prospeo', 'Findymail', 'FullEnrich', 'BetterContact', 'Abstract API', 'Wappalyzer',
          'Meta Ad Library', 'Ads Transparency', 'Foreplay', 'AdSpy', 'BigSpy', 'Panoramata', 'Swipe Files']
def norm(f):
    rec = json.load(open(f)); m = rec['_run']; r = rec['response']; route = m['route']
    out = {'id': m['id'], 'route': route, 'latency_ms': m['latency_ms'], 'fetched_at': m['ts'], 'answer': '', 'sources': [],
           'model': None, 'provider_timestamp': None, 'web_search': None, 'search_queries': None, 'served_by': None}
    if route.startswith('chatgpt-api'):
        t = r['tasks'][0]; res = (t.get('result') or [{}])[0]
        out['answer'] = ''.join(s.get('text') or '' for it in res.get('items') or [] for s in it.get('sections') or [])
        out['sources'] = [a.get('url') for it in res.get('items') or [] for s in it.get('sections') or [] for a in s.get('annotations') or []]
        out.update(model=res.get('model_name'), provider_timestamp=res.get('datetime'), web_search=res.get('web_search'), search_queries=res.get('fan_out_queries'), cost_usd=t.get('cost'))
    elif route == 'perplexity-routed':
        o = r.get('output') or {}; out['answer'] = o.get('answer') or ''
        out['sources'] = [s.get('url') if isinstance(s, dict) else s for s in o.get('sources') or []]
        out['served_by'] = (r.get('_treg') or {}).get('served_by'); out['source_kind'] = o.get('source_kind')
        out['cost_usd'] = ((r.get('_treg') or {}).get('charged_micro') or 0) / 1e6
        try:
            res = r['raw']['tasks'][0]['result'][0]
            out.update(model=res.get('model_name'), provider_timestamp=res.get('datetime'), web_search=res.get('web_search'))
        except Exception: pass
    else:
        res = r.get('result') or {}; out['answer'] = res.get('text') or ''
        out['sources'] = [s.get('url') for s in res.get('sources') or []]
        out['model'] = res.get('model')
        if 'searchQueries' in res: out['search_queries'] = res.get('searchQueries')
    out['sources'] = [re.sub(r'[?&]utm_source=[^&#]*', '', u or '') for u in out['sources']]
    out['n_sources'] = len(out['sources'])
    out['grade'] = grades.get(m['id'], {}).get(route)
    low = out['answer'].lower()
    out['brands'] = [b for b in BRANDS if re.search(r'\b' + re.escape(b.lower()) + r'\b', low)]
    return out
calls, dropped = [], 0
for f in sorted(glob.glob(os.path.join(d, 'raw', '*.json'))):
    c = norm(f)
    if any(t in (c['answer'] + ' '.join(c['sources'])).lower() for t in EXCLUDED):
        dropped += 1; continue
    calls.append(c)
routes = sorted({c['route'] for c in calls})
summary = {}
for rt in routes:
    cs = [c for c in calls if c['route'] == rt]; fresh = [c for c in cs if c['grade']]
    summary[rt] = {
        'calls': len(cs), 'with_sources': sum(c['n_sources'] > 0 for c in cs),
        'fresh_correct': sum(c['grade'] == 'correct' for c in fresh), 'fresh_graded': len(fresh),
        'web_search_reported_true': sum(c['web_search'] is True for c in cs) if any(c['web_search'] is not None for c in cs) else None,
        'returns_model': any(c['model'] for c in cs), 'returns_timestamp': any(c['provider_timestamp'] for c in cs),
        'returns_search_queries': sum(bool(c['search_queries']) for c in cs),
        'median_latency_ms': sorted(c['latency_ms'] for c in cs)[len(cs) // 2],
        'cost_usd_total': round(sum(c.get('cost_usd') or 0 for c in cs), 6) if any('cost_usd' in c for c in cs) else None,
    }
# panel prompts: how many of the brands a live answer named does the model-only answer also name
overlap = {}
for p in [p for p in fx['prompts'] if p['kind'] == 'panel']:
    by = {c['route']: set(c['brands']) for c in calls if c['id'] == p['id']}
    live = by.get('chatgpt-web', set()); base = by.get('chatgpt-api-nosearch', set())
    overlap[p['id']] = {'chatgpt_web': sorted(live), 'chatgpt_api_nosearch': sorted(base), 'shared': sorted(live & base)}
res = {'run': os.path.basename(d.rstrip('/')), 'fixture': fx['name'], 'calls_published': len(calls), 'calls_dropped_exclusion': dropped,
       'summary': summary, 'panel_brand_overlap': overlap, 'calls': calls}
json.dump(res, open(os.path.join(d, 'scored.json'), 'w'), indent=1)
for rt, s in summary.items(): print(rt, s)
for k, v in overlap.items(): print(k, len(v['chatgpt_web']), len(v['chatgpt_api_nosearch']), len(v['shared']), v['shared'])