#!/usr/bin/env python3
"""Counts organic results whose main URL field is a google.com/goto link, per SERP route, from stored
raw responses. Also reports which other fields on those rows still name the site, and every JSON path
where a goto link appears. No provider call. Output carries counts and field names, no result URLs.
Usage: scripts/score-goto-check.py <out.json> label=dir [label=dir ...]"""
import json, sys, os, glob, re, collections
from urllib.parse import urlparse
def host(u):
    if not isinstance(u, str) or not u: return ''
    u = u.split(' ')[0]
    if '://' not in u: u = 'https://' + u
    try: h = urlparse(u).netloc.lower()
    except Exception: return ''
    return h[4:] if h.startswith('www.') else h
def is_goto(u): return isinstance(u, str) and re.match(r'https?://(www\.)?google\.[a-z.]+/goto\b', u) is not None
def walk(o, p=''):
    if isinstance(o, dict):
        for k, v in o.items(): yield from walk(v, p + '.' + k)
    elif isinstance(o, list):
        for v in o: yield from walk(v, p + '[]')
    elif isinstance(o, str): yield p, o
def organic(prov, r):
    """Returns the organic rows and the name of the URL field a rank tracker reads."""
    if not isinstance(r, dict): return [], ''
    if prov == 'dataforseo':
        rows = []
        for t in r.get('tasks') or []:
            for res in t.get('result') or []:
                rows += [i for i in res.get('items') or [] if i.get('type') == 'organic']
        return rows, 'url'
    if prov == 'cloro': return ((r.get('result') or {}).get('organicResults') or []), 'link'
    if prov == 'serpapi': return (r.get('organic_results') or []), 'link'
    if prov == 'serper': return (r.get('organic') or []), 'link'
    if prov == 'limadata': return (r.get('organic') or []), 'url'
    if prov == 'scrapecreators': return (r.get('results') or []), 'url'
    if prov == 'anyapi': return (((r.get('output') or {}).get('data') or {}).get('results') or []), 'link'
    if prov in ('treg-routed', 'treg'):
        o = r.get('output') or {}
        rows = o.get('results') or o.get('organic') or []
        return rows, 'url' if rows and 'url' in rows[0] else 'link'
    return [], ''
SITE_FIELDS = ['domain', 'breadcrumb', 'displayed_link', 'displayedLink', 'displayed_url', 'website_name', 'source', 'url', 'link']
def score(d):
    by = collections.defaultdict(lambda: {'calls': 0, 'with_page': 0, 'failed': 0, 'organic': 0, 'goto': 0, 'calls_with_goto': 0, 'empty_url': 0, 'other_google': 0, 'site_fields': collections.Counter(), 'goto_paths': collections.Counter(), 'served_by': collections.Counter(), 'by_q': {}})
    for f in sorted(glob.glob(os.path.join(d, 'serp-*.json'))):
        rec = json.load(open(f)); run = rec['_run']; prov = run['provider']
        if prov == 'serpstat': continue
        r = rec['response']; b = by[prov]; b['calls'] += 1
        tr = r.get('_treg') if isinstance(r, dict) else None
        if tr: b['served_by'][tr.get('served_by') or '?'] += 1
        rows, field = organic(prov, r)
        for p, s in walk(r):
            if 'google.com/goto' in s: b['goto_paths'][re.sub(r'^\.', '', p)] += 1
        if not rows: b['failed'] += 1; continue
        b['with_page'] += 1; b['organic'] += len(rows); n = 0
        for row in rows:
            u = row.get(field)
            if not u: b['empty_url'] += 1
            elif is_goto(u):
                n += 1
                for sf in SITE_FIELDS:
                    if sf == field: continue
                    v = row.get(sf)
                    if v and host(v) and 'google.' not in host(v) and '.' in host(v): b['site_fields'][sf] += 1
            elif host(u).startswith('google.'): b['other_google'] += 1
        b['goto'] += n; b['calls_with_goto'] += 1 if n else 0
        b['by_q'][run.get('q', f)] = [n, len(rows)]
    out = {}
    for k, b in sorted(by.items()):
        out[k] = {**{x: b[x] for x in ('calls', 'with_page', 'failed', 'organic', 'goto', 'calls_with_goto', 'empty_url', 'other_google')},
                  'goto_share': round(b['goto'] / b['organic'], 4) if b['organic'] else None,
                  'site_fields_on_goto_rows': dict(b['site_fields']), 'goto_paths': dict(b['goto_paths'].most_common(15)),
                  'served_by': dict(b['served_by']), 'by_query': b['by_q']}
    return out
res = {a.split('=', 1)[0]: score(a.split('=', 1)[1]) for a in sys.argv[2:]}
json.dump(res, open(sys.argv[1], 'w'), indent=1, ensure_ascii=False)
for lab, s in res.items():
    print('==', lab)
    for k, v in s.items():
        print(f"{k:15} calls {v['calls']:3} page {v['with_page']:3} organic {v['organic']:4} goto {v['goto']:4} ({(v['goto_share'] or 0)*100:.1f}%) callsWith {v['calls_with_goto']:3} otherGoogle {v['other_google']} fields {v['site_fields_on_goto_rows']} served {v['served_by']}")
