#!/usr/bin/env python3
"""Runs each prompt of an AI-answer fixture through seven answer routes: the live ChatGPT web interface
with and without its disableWebSearch flag (Cloro), Google AI Mode and Gemini (Cloro), Treg's routed
Perplexity endpoint, and the ChatGPT model API with web search off and on (DataForSEO LLM Responses).
One raw response per call with wall-clock latency, same record format as run-serp-routes.py.
Deterministic; no LLM of our own.
Usage: scripts/run-ai-answer-check.py evidence/fixtures/ai-answer-check-v1.json evidence/ai-answer-checks/YYYY-MM-DD
Cost on 2026-10-08 list prices for v1: about $0.80 for 70 calls (the two DataForSEO API routes are $0.58 of it)."""
import json, sys, os, subprocess, time, concurrent.futures as cf
fx = json.load(open(sys.argv[1])); out = sys.argv[2]; os.makedirs(out, exist_ok=True)
def call(label, argv, meta):
    f = os.path.join(out, label + '.json')
    if os.path.exists(f): return
    t0 = time.time()
    p = subprocess.run(['treg', 'call', *argv], capture_output=True, text=True)
    ms = int((time.time() - t0) * 1000)
    raw = p.stdout if p.stdout.strip() else p.stderr
    try: body = json.loads(raw)
    except Exception: body = {'_nonjson': raw[:2000]}
    rec = {'_run': {**meta, 'label': label, 'latency_ms': ms, 'exit': p.returncode, 'ts': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}, 'response': body}
    json.dump(rec, open(f, 'w'))
    print(label, ms, 'ms', 'exit', p.returncode, flush=True)
def dfs(prompt, web):
    return json.dumps([{'user_prompt': prompt, 'model_name': 'gpt-4.1-mini', 'web_search': web, 'max_output_tokens': 1200, **({'web_search_country_iso_code': 'US'} if web else {})}])
jobs = []
for p in fx['prompts']:
    k, q = p['id'], p['prompt']
    jobs.append((f'chatgpt-web-{k}', ['cloro.ai-search.chatgpt.scrape', '--method', 'POST', '--data', json.dumps({'prompt': q, 'country': 'US', 'include': {'searchQueries': True}})], {'route': 'chatgpt-web', 'id': k}))
    jobs.append((f'chatgpt-web-nosearch-{k}', ['cloro.ai-search.chatgpt.scrape', '--method', 'POST', '--data', json.dumps({'prompt': q, 'country': 'US', 'disableWebSearch': True, 'include': {'searchQueries': True}})], {'route': 'chatgpt-web-nosearch', 'id': k}))
    jobs.append((f'aimode-{k}', ['cloro.google.serp.ai_mode', '--method', 'POST', '--data', json.dumps({'prompt': q, 'gl': 'US'})], {'route': 'aimode', 'id': k}))
    jobs.append((f'gemini-{k}', ['cloro.ai-search.gemini.scrape', '--method', 'POST', '--data', json.dumps({'prompt': q, 'country': 'US'})], {'route': 'gemini', 'id': k}))
    jobs.append((f'perplexity-routed-{k}', ['treg.ai-search.perplexity.answer', '--method', 'POST', '--data', json.dumps({'prompt': q})], {'route': 'perplexity-routed', 'id': k}))
    jobs.append((f'chatgpt-api-nosearch-{k}', ['dataforseo.x.ai-optimization-chat-gpt-llm-responses-live', '--method', 'POST', '--data', dfs(q, False)], {'route': 'chatgpt-api-nosearch', 'id': k}))
    jobs.append((f'chatgpt-api-search-{k}', ['dataforseo.x.ai-optimization-chat-gpt-llm-responses-live', '--method', 'POST', '--data', dfs(q, True)], {'route': 'chatgpt-api-search', 'id': k}))
# one worker per route so latency is not distorted by our own concurrency
by_route = {}
for j in jobs: by_route.setdefault(j[2]['route'], []).append(j)
def run_route(lst):
    for label, argv, meta in lst: call(label, argv, meta)
with cf.ThreadPoolExecutor(max_workers=len(by_route)) as ex:
    list(ex.map(run_route, by_route.values()))
print('done', len(jobs), 'jobs')