#!/usr/bin/env python3 """Pre-registered analysis for the tau3 banking_knowledge replication (docs/bench/tau3-confirm-1101/PLAN.md). usage: analyze.py --d300 a.json --g300 b.json --g301 c.json --d301 d.json [--rows] Unit: a (task, seed) pair that finished in BOTH arms of that seed. A simulation with termination_reason infrastructure_error does not count as finished. If a task has more than one simulation in a file, the last one counts. Cost: per-call usage at list price (summarize.py msg_cost). Agent cost is the primary metric; user-simulator cost is reported but is not part of it (both arms call the user simulator directly, with no gateway). Primary: pooled agent-cost change sum(G)/sum(D) - 1 over all pairs; 95% CI from a paired bootstrap that resamples TASKS (a task carries all of its seed pairs), 10,000 draws, random.Random(1), percentile 2.5 / 97.5. Pass: per seed, exact two-sided McNemar on discordant pairs. Pooled: per-task difference d_t = mean over that task's seeds of (gateway pass - direct pass); 95% CI of mean(d_t) from 10,000 task resamples, random.Random(1). A significant drop means the CI upper bound is below 0. Claim bar (fixed before any paid run): BEYOND_INDUCTION_37 is true only if the pooled agent-cost CI upper bound is below -0.37 AND there is no significant pooled pass drop.""" import json, sys, math, random, os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from summarize import msg_cost, wilson N_BOOT = 10000 BAR = -0.37 def load(path): by = {} for s in json.load(open(path))['simulations']: if s.get('termination_reason') == 'infrastructure_error': continue acc = dict(i=0, cw=0, cr=0, o=0, usd=0.0, calls=0, long=0, uusd=0.0) for m in s.get('messages') or []: u = (m.get('raw_data') or {}).get('usage') if not u: continue c = msg_cost(u) if m.get('role') == 'assistant': for k in ('i', 'cw', 'cr', 'o', 'usd', 'long'): acc[k] += c[k] acc['calls'] += 1 elif m.get('role') == 'user': acc['uusd'] += c['usd'] acc['pass'] = int(((s.get('reward_info') or {}).get('reward') or 0) >= 1 - 1e-9) acc['term'] = s.get('termination_reason') by[s['task_id']] = acc return by def mcnemar(b, c): n = b + c if n == 0: return 1.0 k = min(b, c) return min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) def pct(xs, q): return xs[int(q * len(xs))] def boot_cost(pairs, key='usd'): """pairs: list of (task, D_row, G_row). Resample tasks, keep each task's seed pairs together.""" tasks = sorted({t for t, _, _ in pairs}) per = {t: [0.0, 0.0] for t in tasks} for t, d, g in pairs: per[t][0] += d[key]; per[t][1] += g[key] a = [per[t][0] for t in tasks]; b = [per[t][1] for t in tasks] r = random.Random(1); k = len(tasks); out = [] for _ in range(N_BOOT): ix = [r.randrange(k) for _ in range(k)] sa = sum(a[i] for i in ix) out.append(sum(b[i] for i in ix) / sa - 1 if sa else 0.0) out.sort() return round(sum(b) / sum(a) - 1, 4), [round(pct(out, 0.025), 4), round(pct(out, 0.975), 4)] def boot_pass(pairs): per = {} for t, d, g in pairs: per.setdefault(t, []).append(g['pass'] - d['pass']) tasks = sorted(per); dt = [sum(per[t]) / len(per[t]) for t in tasks] r = random.Random(1); k = len(tasks); out = [] for _ in range(N_BOOT): ix = [r.randrange(k) for _ in range(k)] out.append(sum(dt[i] for i in ix) / k) out.sort() return round(sum(dt) / k, 4), [round(pct(out, 0.025), 4), round(pct(out, 0.975), 4)] def arm(X, ts): n = len(ts); k = sum(X[t]['pass'] for t in ts) return dict(passed=k, n=n, pass_rate=round(k / n, 4) if n else None, ci95=[round(x, 4) for x in wilson(k, n)], agent_usd=round(sum(X[t]['usd'] for t in ts), 4), user_usd=round(sum(X[t]['uusd'] for t in ts), 4), tokens={c: sum(X[t][c] for t in ts) for c in ('i', 'cw', 'cr', 'o')}, calls=sum(X[t]['calls'] for t in ts), long_calls=sum(X[t]['long'] for t in ts)) def seed_block(D, G): ts = sorted(set(D) & set(G)) pairs = [(t, D[t], G[t]) for t in ts] res = dict(paired=len(ts), direct=arm(D, ts), gateway=arm(G, ts)) if ts: res['agent_cost_change'], res['agent_cost_change_ci95'] = boot_cost(pairs) b = sum(1 for t in ts if D[t]['pass'] and not G[t]['pass']); c = sum(1 for t in ts if G[t]['pass'] and not D[t]['pass']) res['discordant'] = dict(direct_only=b, gateway_only=c, direct_only_tasks=[t for t in ts if D[t]['pass'] and not G[t]['pass']], gateway_only_tasks=[t for t in ts if G[t]['pass'] and not D[t]['pass']]) res['mcnemar_exact_p'] = round(mcnemar(b, c), 4) return res, pairs def main(): a = sys.argv[1:] f = {k: a[a.index('--' + k) + 1] for k in ('d300', 'g300', 'g301', 'd301') if '--' + k in a} out = dict(files={k: os.path.basename(os.path.dirname(v)) for k, v in f.items()}) pooled = [] rows = [] for s in ('300', '301'): if 'd' + s not in f or 'g' + s not in f: out['seed_' + s] = None continue D, G = load(f['d' + s]), load(f['g' + s]) blk, pairs = seed_block(D, G) blk['finished_direct'] = len(D); blk['finished_gateway'] = len(G) out['seed_' + s] = blk pooled += pairs rows += [dict(seed=int(s), task=t, d_pass=d['pass'], g_pass=g['pass'], d_usd=round(d['usd'], 5), g_usd=round(g['usd'], 5), d_calls=d['calls'], g_calls=g['calls'], d_in=d['i'], g_in=g['i'], d_cw=d['cw'], g_cw=g['cw']) for t, d, g in pairs] P = dict(pairs=len(pooled), tasks=len({t for t, _, _ in pooled})) if pooled: P['agent_usd_direct'] = round(sum(d['usd'] for _, d, _ in pooled), 4) P['agent_usd_gateway'] = round(sum(g['usd'] for _, _, g in pooled), 4) P['user_usd_direct'] = round(sum(d['uusd'] for _, d, _ in pooled), 4) P['user_usd_gateway'] = round(sum(g['uusd'] for _, _, g in pooled), 4) P['agent_cost_change'], P['agent_cost_change_ci95'] = boot_cost(pooled) P['input_tokens_change'], P['input_tokens_change_ci95'] = boot_cost(pooled, 'i') P['passed_direct'] = sum(d['pass'] for _, d, _ in pooled); P['passed_gateway'] = sum(g['pass'] for _, _, g in pooled) P['pass_diff_per_task'], P['pass_diff_ci95'] = boot_pass(pooled) P['significant_pass_drop'] = P['pass_diff_ci95'][1] < 0 P['BEYOND_INDUCTION_37'] = bool(P['agent_cost_change_ci95'][1] < BAR and not P['significant_pass_drop']) out['pooled'] = P if '--rows' in a: out['rows'] = rows print(json.dumps(out, indent=1)) if __name__ == '__main__': main()