#!/usr/bin/env python3
"""Groups the cohort's 50 queries by SERP overlap, the way a SERP-based keyword clusterer does:
two keywords are one intent when their first ten organic URLs share at least N URLs (N = 3, 4, 5).
Groups are connected components of those links. Computed per route and per run, then compared:
across routes on the same run, the same route on runs 30 minutes apart, and the same route
across dates. Reads the raw responses already stored by the cohort and SERP runs; no provider call.
Writes summary.json and CSV tables to <out-dir>; the tables hold query text and counts only, no
result URLs, so nothing from a result page is republished.
Usage: scripts/score-serp-clusters.py evidence/fixtures/seo-cohort-v1.json <out-dir> <label>=<run-dir> [...]"""
import json, sys, os, glob, itertools, collections, csv, statistics

cohort = json.load(open(sys.argv[1])); out_dir = sys.argv[2].rstrip('/')
runs_arg = [a.split('=', 1) for a in sys.argv[3:]]
Q = cohort['queries']
# shared parsing with the SERP-check scorer, so URLs are read and normalised the same way
sys.argv = [sys.argv[0], sys.argv[1], '.', '.']
exec(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'score-serp-check.py')).read().split('# ---------- per route ----------')[0])
THRESHOLDS = (3, 4, 5)
# USD per call used for the cost-per-1,000 column; source of each figure alongside
COST = {
    'dataforseo': (0.002, 'cost field in each response (live advanced)'),
    'cloro': (0.002, 'settled charge on the account ledger, 2026-09-28'),
    'serpapi': (0.015, 'catalog list price, 2026-09-28'),
    'serper': (0.001, 'settled charge on the account ledger, 2026-09-28'),
    'anyapi': (0.0004, 'settled charge on the account ledger, 2026-09-28'),
    'limadata': (0.002, 'settled charge on the account ledger, 2026-09-28'),
    'scrapecreators': (0.00188, 'catalog list price for one credit, 2026-09-28'),
    'treg-routed': (0.0, '$0 charged by Treg; served by ScrapeCreators on a key registered on the account, one credit a call'),
    'serpstat': (0.0096, 'about 19 rows billed at $0.0005 a row per answered call; answers only queries in its stored index'),
}

def load_run(d):
    by = collections.defaultdict(dict)
    for f in sorted(glob.glob(os.path.join(d, 'serp-*-q[0-9][0-9].json'))):
        r = json.load(open(f))
        if r['_run'].get('task') != 'serp' or not ok(r): continue
        u = urls(r)
        if u: by[r['_run']['provider']][r['_run']['q']] = set(x for x in u if x)
    return by

def links(m, n): return {(a, b) for a, b in itertools.combinations(Q, 2) if a in m and b in m and len(m[a] & m[b]) >= n}
def groups(m, n):
    parent = {q: q for q in m}
    def find(x):
        while parent[x] != x: parent[x] = parent[parent[x]]; x = parent[x]
        return x
    for a, b in links(m, n): parent[find(a)] = find(b)
    g = collections.defaultdict(list)
    for q in Q:
        if q in m: g[find(q)].append(q)
    return sorted((v for v in g.values() if len(v) > 1), key=lambda v: (-len(v), Q.index(v[0])))
def jac(a, b): return round(len(a & b) / len(a | b), 2) if a | b else None
def partition_agree(ga, gb, qs):
    """share of query pairs (both keywords answered by both routes) that the two groupings treat the same way: together in both or apart in both"""
    def lab(g):
        d = {}
        for i, grp in enumerate(g):
            for q in grp: d[q] = i
        return d
    la, lb = lab(ga), lab(gb); same = tot = 0
    for a, b in itertools.combinations(qs, 2):
        ta = a in la and b in la and la[a] == la[b]; tb = a in lb and b in lb and lb[a] == lb[b]
        if ta or tb:
            tot += 1; same += ta == tb
    return round(same / tot, 2) if tot else None

runs = {label: load_run(d) for label, d in runs_arg}
rows, grp_rows, summary = [], [], {'cohort': cohort['version'], 'market': cohort['market'], 'thresholds': THRESHOLDS,
    'runs': {l: d for l, d in runs_arg}, 'routes': {}, 'cost': {}}
for label, by in runs.items():
    for p in [x for x in ORDER if x in by]:
        m = by[p]; row = {'run': label, 'route': p, 'queries_answered': len(m)}
        for n in THRESHOLDS:
            g = groups(m, n)
            row[f'links_{n}'] = len(links(m, n)); row[f'groups_{n}'] = len(g); row[f'keywords_grouped_{n}'] = sum(len(x) for x in g)
            row[f'largest_group_{n}'] = max((len(x) for x in g), default=0)
        rows.append(row); summary['routes'].setdefault(p, {})[label] = row
        for i, g in enumerate(groups(m, 5)):
            grp_rows.append({'run': label, 'route': p, 'group': i + 1, 'size': len(g), 'keywords': ' | '.join(g)})

def compare(pairs_spec, n):
    out = []
    for (la, pa), (lb, pb) in pairs_spec:
        A, B = runs[la].get(pa), runs[lb].get(pb)
        if not A or not B: continue
        qs = [q for q in Q if q in A and q in B]
        A2, B2 = {q: A[q] for q in qs}, {q: B[q] for q in qs}
        la_, lb_ = links(A2, n), links(B2, n)
        out.append({'a': f'{pa} @ {la}', 'b': f'{pb} @ {lb}', 'threshold': n, 'queries_both': len(qs), 'links_a': len(la_), 'links_b': len(lb_),
                    'links_shared': len(la_ & lb_), 'link_jaccard': jac(la_, lb_), 'grouping_agreement': partition_agree(groups(A2, n), groups(B2, n), qs)})
    return out

labels = [l for l, _ in runs_arg]
cmp_rows = []
for n in THRESHOLDS:
    # same run, every pair of routes
    for label in labels:
        ps = [p for p in ORDER if p in runs[label]]
        cmp_rows += [dict(kind='same run, two routes', **r) for r in compare([((label, a), (label, b)) for a, b in itertools.combinations(ps, 2)], n)]
    # same route, every pair of runs
    for p in ORDER:
        have = [l for l in labels if p in runs[l]]
        cmp_rows += [dict(kind='same route, two runs', **r) for r in compare([((a, p), (b, p)) for a, b in itertools.combinations(have, 2)], n)]

# links every live route found on one run
consensus = {}
for label in labels:
    live = [p for p in ORDER if p in runs[label] and p != 'serpstat']
    sets_ = {p: links(runs[label][p], 5) for p in live}
    cnt = collections.Counter(x for s in sets_.values() for x in s)
    consensus[label] = {'routes': live, 'links_any': len(cnt), 'links_all': sum(1 for v in cnt.values() if v == len(live)),
                        'links_one_route_only': sum(1 for v in cnt.values() if v == 1),
                        'by_count': dict(sorted(collections.Counter(cnt.values()).items()))}
summary['consensus_5'] = consensus
for p, (c, src) in COST.items(): summary['cost'][p] = {'usd_per_call': c, 'usd_per_1000_keywords': round(c * 1000, 2), 'source': src}
summary['comparisons'] = cmp_rows

os.makedirs(out_dir, exist_ok=True)
json.dump(summary, open(os.path.join(out_dir, 'summary.json'), 'w'), indent=1)
def write(name, rs):
    if not rs: return
    with open(os.path.join(out_dir, name), 'w', newline='') as f:
        w = csv.DictWriter(f, fieldnames=list(rs[0].keys())); w.writeheader(); w.writerows(rs)
# shared top-10 URLs for four keyword pairs any clusterer should put together
EXAMPLES = [('backlinks api', 'backlink api'), ('keyword research api', 'keyword search volume api'), ('clay alternatives', 'clay alternatives for developers'), ('meta ad library api', 'ad library api')]
ex_rows = [{'run': l, 'route': p, 'keyword_a': a, 'keyword_b': b, 'shared_top10_urls': len(runs[l][p][a] & runs[l][p][b])}
           for a, b in EXAMPLES for l in labels for p in ORDER if p in runs[l] and a in runs[l][p] and b in runs[l][p]]
write('example-pairs.csv', ex_rows)
write('routes.csv', rows); write('groups-5.csv', grp_rows); write('comparisons.csv', cmp_rows)
print(json.dumps({'consensus_5': consensus}, indent=1))