#!/usr/bin/env python3
"""Prices one client SEO audit on pay-per-call routes: rankings for the cohort's 50 queries, volumes
for the same 50, the domain's ranked keywords, its backlink profile and a 100-page on-page crawl,
through DataForSEO (live) and Serpstat (stored index). One raw response per call with wall-clock
latency, the balance before and after, and the Treg ledger for the run window (polled every 30 s, since the CLI caps a read at the last 200 rows). Deterministic; no LLM.
Usage: scripts/run-audit-cost.py evidence/fixtures/seo-cohort-v1.json evidence/audit-cost/YYYY-MM-DD --domain hunter.io
Resumable: a call whose raw file exists is skipped. Cost on 2026-10-29 list prices: about $1.5 for one domain."""
import json, sys, os, subprocess, time, threading, concurrent.futures as cf
args = sys.argv[1:]
domain = args[args.index('--domain') + 1] if '--domain' in args else 'hunter.io'
pos = [a for i, a in enumerate(args) if a != '--domain' and (i == 0 or args[i - 1] != '--domain')]
cohort = json.load(open(pos[0])); out = pos[1]; raw = os.path.join(out, 'raw'); os.makedirs(raw, exist_ok=True)
LOC = cohort['market']['location_code']; Q = cohort['queries']
def treg(argv, timeout=180):
    try:
        p = subprocess.run(['treg', *argv], capture_output=True, text=True, timeout=timeout)
        return p.returncode, (p.stdout if p.stdout.strip() else p.stderr)
    except subprocess.TimeoutExpired:
        return -1, 'timeout'
def balance():
    rc, s = treg(['balance', '--json'], 60)
    try: return json.loads(s)['balance_usd']
    except Exception: return None
def call(label, argv, meta):
    f = os.path.join(raw, label + '.json')
    if os.path.exists(f): return json.load(open(f))
    t0 = time.time(); rc, s = treg(['call', *argv]); ms = int((time.time() - t0) * 1000)
    try: body = json.loads(s)
    except Exception: body = {'_nonjson': s[:2000]}
    rec = {'_run': {**meta, 'label': label, 'latency_ms': ms, 'exit': rc, 'ts': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}, 'response': body}
    json.dump(rec, open(f, 'w')); print(label, ms, 'ms', 'exit', rc, flush=True); return rec
def dfs(ep, task): return [ep, '--method', 'POST', '--data', json.dumps([task])]
def sst(ep, method, params): return [ep, '--method', 'POST', '--data', json.dumps({'id': '1', 'method': method, 'params': params})]
jobs = {'dataforseo': [], 'serpstat': []}
for i, q in enumerate(Q):
    k = f'q{i+1:02d}'
    jobs['dataforseo'].append((f'serp-dataforseo-{k}', dfs('dataforseo.x.serp-google-organic-live-advanced', {'language_code': 'en', 'location_code': LOC, 'keyword': q, 'depth': 10}), {'task': 'serp', 'provider': 'dataforseo', 'q': q}))
    jobs['serpstat'].append((f'serp-serpstat-{k}', sst('serpstat.google.serp.organic', 'SerpstatKeywordProcedure.getKeywordTop', {'keyword': q, 'se': 'g_us', 'size': 10}), {'task': 'serp', 'provider': 'serpstat', 'q': q}))
jobs['dataforseo'] += [
    ('kw-dataforseo-all', dfs('dataforseo.google.keywords.volume', {'keywords': Q, 'location_code': LOC, 'language_code': 'en'}), {'task': 'kw', 'provider': 'dataforseo', 'n': len(Q)}),
    ('rk-dataforseo', dfs('dataforseo.google.domain.ranked_keywords', {'target': domain, 'location_code': LOC, 'language_code': 'en', 'limit': 1000, 'load_rank_absolute': True, 'order_by': ['ranked_serp_element.serp_item.rank_group,asc']}), {'task': 'rk', 'provider': 'dataforseo', 'd': domain}),
    ('bl-dataforseo', dfs('dataforseo.web.backlinks.summary', {'target': domain}), {'task': 'bl', 'provider': 'dataforseo', 'd': domain}),
    ('rd-dataforseo', dfs('dataforseo.web.linking_domains.list', {'target': domain, 'limit': 100, 'order_by': ['rank,desc']}), {'task': 'rd', 'provider': 'dataforseo', 'd': domain}),
]
jobs['serpstat'] += [
    ('kw-serpstat-all', sst('serpstat.google.keywords.volume', 'SerpstatKeywordProcedure.getKeywordsInfo', {'keywords': Q, 'se': 'g_us'}), {'task': 'kw', 'provider': 'serpstat', 'n': len(Q)}),
    ('rk-serpstat', sst('serpstat.google.domain.ranked_keywords', 'SerpstatDomainProcedure.getDomainKeywords', {'domain': domain, 'se': 'g_us', 'size': 1000, 'sort': {'position': 'asc'}}), {'task': 'rk', 'provider': 'serpstat', 'd': domain}),
    ('rkq-serpstat', sst('serpstat.google.domain.ranked_keywords', 'SerpstatDomainProcedure.getDomainKeywords', {'domain': domain, 'se': 'g_us', 'size': 100, 'keywords': Q}), {'task': 'rkq', 'provider': 'serpstat', 'd': domain, 'n': len(Q)}),
    ('bl-serpstat', sst('serpstat.web.backlinks.summary', 'SerpstatBacklinksProcedure.getSummaryV2', {'query': domain}), {'task': 'bl', 'provider': 'serpstat', 'd': domain}),
    ('ov-serpstat', sst('serpstat.google.domain.overview', 'SerpstatDomainProcedure.getDomainsInfo', {'domains': [domain], 'se': 'g_us'}), {'task': 'ov', 'provider': 'serpstat', 'd': domain}),
]
def run_onpage():
    post = call('onpage-dataforseo-post', dfs('dataforseo.x.on-page-task-post', {'target': domain, 'max_crawl_pages': 100, 'load_resources': False, 'enable_javascript': False}), {'task': 'onpage', 'provider': 'dataforseo', 'd': domain, 'pages': 100})
    try: tid = post['response']['tasks'][0]['id']
    except Exception: print('onpage post failed', flush=True); return
    f = os.path.join(raw, 'onpage-dataforseo-summary.json')
    if os.path.exists(f): return
    deadline = time.time() + 900; last = None
    while time.time() < deadline:
        time.sleep(30)
        t0 = time.time(); rc, s = treg(['call', 'dataforseo.x.on-page-summary-id', '--query', f'id={tid}']); ms = int((time.time() - t0) * 1000)
        try: body = json.loads(s)
        except Exception: body = {'_nonjson': s[:2000]}
        last = {'_run': {'task': 'onpage-summary', 'provider': 'dataforseo', 'd': domain, 'task_id': tid, 'label': 'onpage-dataforseo-summary', 'latency_ms': ms, 'exit': rc, 'ts': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}, 'response': body}
        try:
            prog = body['tasks'][0]['result'][0]['crawl_progress']; print('onpage', prog, body['tasks'][0]['result'][0]['crawl_status'], flush=True)
            if prog == 'finished': break
        except Exception: print('onpage poll', rc, str(body)[:200], flush=True)
    if last: json.dump(last, open(f, 'w'))
ledger = {}; stop = threading.Event()
def poll_ledger():
    rc, s = treg(['balance', '--limit', '200', '--json'], 90)
    try:
        for e in json.loads(s)['entries']['items']: ledger[e['id']] = e
    except Exception: print('ledger poll failed', s[:120], flush=True)
def poll_loop():
    while not stop.wait(30): poll_ledger()
poller = threading.Thread(target=poll_loop, daemon=True); poller.start()
t_start = time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime()); b0 = balance(); print('balance before', b0, t_start, flush=True)
def run_prov(lst):
    for label, argv, meta in lst: call(label, argv, meta)
with cf.ThreadPoolExecutor(max_workers=3) as ex:
    futs = [ex.submit(run_prov, jobs['serpstat']), ex.submit(run_onpage)]
    run_prov(jobs['dataforseo'])  # DataForSEO SERP and keyword calls stay sequential: the rate limit is per account
    for f in futs: f.result()
time.sleep(20); b1 = balance(); t_end = time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime()); print('balance after', b1, t_end, flush=True)
stop.set(); poller.join(); poll_ledger()
win = [e for e in ledger.values() if e['created_at'] >= t_start]
json.dump({'domain': domain, 'started': t_start, 'ended': t_end, 'balance_before': b0, 'balance_after': b1, 'entries': sorted(win, key=lambda e: e['created_at'])}, open(os.path.join(out, 'ledger.json'), 'w'), indent=1)
print('done; ledger entries in window', len(win), flush=True)