#!/usr/bin/env python3
"""Follows a fixed-seed sample of google.com/goto links from stored SERP responses, one request a second,
without loading the destination page (stops at the first non-Google host), and checks whether the row's
other fields named the same site. No provider call; plain HTTP from one IP. Per-link results go to
<raw-dir>/resolved.json (gitignored); the summary carries counts only.
Usage: scripts/resolve-goto-sample.py <raw-dir> <summary.json> [per-route-sample]"""
import json, sys, os, glob, random, re, time, collections, http.client
from urllib.parse import urlparse, urljoin
raw, out = sys.argv[1], sys.argv[2]; N = int(sys.argv[3]) if len(sys.argv) > 3 else 60
UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36'
def host(u):
    if not isinstance(u, str) or not u: return ''
    u = u.split(' ')[0]
    if '://' not in u: u = 'https://' + u
    try: h = urlparse(u).netloc.lower()
    except Exception: return ''
    return h[4:] if h.startswith('www.') else h
def norm(h):
    return '.'.join(h.split('.')[-2:]) if h else ''
is_goto = lambda u: isinstance(u, str) and re.match(r'https?://(www\.)?google\.[a-z.]+/goto\b', u)
rows = {'dataforseo': [], 'serpapi': []}
for f in sorted(glob.glob(os.path.join(raw, 'serp-dataforseo-*.json'))):
    r = json.load(open(f))['response']
    for t in r.get('tasks') or []:
        for res in t.get('result') or []:
            for i in res.get('items') or []:
                if i.get('type') == 'organic' and is_goto(i.get('url')): rows['dataforseo'].append({'u': i['url'], 'domain': i.get('domain'), 'breadcrumb': i.get('breadcrumb'), 'website_name': i.get('website_name')})
for f in sorted(glob.glob(os.path.join(raw, 'serp-serpapi-*.json'))):
    r = json.load(open(f))['response']
    for i in r.get('organic_results') or []:
        if is_goto(i.get('link')): rows['serpapi'].append({'u': i['link'], 'displayed_link': i.get('displayed_link'), 'source': i.get('source')})
random.seed(20261102)
def fetch(u):
    p = urlparse(u); c = http.client.HTTPSConnection(p.netloc, timeout=20)
    c.request('GET', p.path + ('?' + p.query if p.query else ''), headers={'User-Agent': UA}); r = c.getresponse()
    loc = r.getheader('Location'); s = r.status; c.close(); return s, loc
res = []; out_summary = {}
for route, lst in rows.items():
    sample = random.sample(lst, min(N, len(lst)))
    for row in sample:
        u = row['u']; t0 = time.time(); statuses = []; final = None
        try:
            for _ in range(4):
                s, loc = fetch(u); statuses.append(s)
                if s in (301, 302, 303, 307, 308) and loc:
                    u = urljoin(u, loc)
                    if not host(u).startswith('google.'): final = u; break
                else: break
        except Exception as e:
            statuses.append(type(e).__name__)
        res.append({'route': route, **{k: v for k, v in row.items() if k != 'u'}, 'statuses': statuses, 'final': final, 'ms': int((time.time() - t0) * 1000)})
        time.sleep(1)
    got = [x for x in res if x['route'] == route]; ok = [x for x in got if x['final']]
    fields = [k for k in ('domain', 'breadcrumb', 'website_name', 'displayed_link', 'source') if k in row]
    fm = {}
    for k in fields:
        match = sum(1 for x in ok if norm(host(x.get(k))) == norm(host(x['final'])))
        fm[k] = match
    out_summary[route] = {'goto_rows_in_run': len(lst), 'sampled': len(got), 'resolved': len(ok), 'not_resolved_statuses': dict(collections.Counter(str(x['statuses'][-1]) for x in got if not x['final'])),
                          'field_names_same_site': fm, 'median_ms': sorted(x['ms'] for x in got)[len(got) // 2] if got else None}
json.dump(res, open(os.path.join(raw, 'resolved.json'), 'w'), indent=0)
json.dump(out_summary, open(out, 'w'), indent=1)
print(json.dumps(out_summary, indent=1))
