{
 "schema": "tau3-confirm/1",
 "benchmark": "tau3-bench banking_knowledge",
 "set": "tau3-confirm-1101",
 "date": "2026-11-01",
 "status": "pre-registered, audited",
 "context_mode": {
  "model": "claude-haiku-5-5",
  "model_name": "Haiku 5.5",
  "effort": "default",
  "role": "agent and user simulator",
  "tasks": 96,
  "seeds": [
   300,
   301
  ],
  "pairs": 170,
  "harness": {
   "repo": "https://github.com/sierra-research/tau2-bench",
   "commit": "122e2432b82d740e8f313f46f0a071729c404f39",
   "domain": "banking_knowledge",
   "retrieval": "bm25"
  },
  "excluded": [
   "task_102 (it calls OpenAI)"
  ],
  "cost_basis": "Agent cost: the tokens Anthropic returned on each agent call, at list price. The user simulator is counted apart.",
  "agent_cost_change_pct": -39.3,
  "ci_low": -49.5,
  "ci_high": -27.1,
  "agent_usd": {
   "direct": 8.51,
   "gateway": 5.16
  },
  "per_seed": [
   {
    "seed": 300,
    "pairs": 96,
    "change_pct": -42.0,
    "ci_low": -52.9,
    "ci_high": -27.6
   },
   {
    "seed": 301,
    "pairs": 74,
    "change_pct": -35.4,
    "ci_low": -50.7,
    "ci_high": -16.1
   }
  ],
  "pass": {
   "direct": 36,
   "gateway": 33,
   "of": 170,
   "diff_pp": -1.6,
   "diff_ci_low_pp": -6.8,
   "diff_ci_high_pp": 3.7,
   "significant_drop": false
  },
  "notes": [
   "The upper end of our 95% range (-27.1%) is above Induction's -37%, so the ranges overlap.",
   "The gateway's own meter reads 3.0% to 3.3% higher than the client sum. On that meter the change is -37.4%.",
   "Most of the saving comes from fewer calls above 100k prompt tokens, which Anthropic prices higher. At one flat price for every call the change is -9.7%.",
   "Seed 301 direct stopped at 74 of 96 tasks (spend cap)."
  ],
  "next_run": {
   "text": "We run the same model, Haiku 5.5, at higher reasoning effort next.",
   "source": "Owner plan, 2026-11-02"
  }
 },
 "headline": "-39% vs Induction's published -37% (medium)",
 "models_differ": "The models differ: Induction used gpt-5.6-sol, we used Claude Haiku 5.5.",
 "induction": {
  "read_on": "2026-11-01",
  "source": "https://www.induction.ai/docs/benchmarks/tau3",
  "method": "https://www.induction.ai/docs/benchmarks",
  "model": "gpt-5.6-sol",
  "tasks": 97,
  "rows": [
   {
    "effort": "medium",
    "cost_change_pct": -37,
    "usd_per_run": {
     "direct": 59,
     "proxy": 37
    },
    "pass_pct": {
     "direct": 28.6,
     "proxy": 31.3
    },
    "tier_match": true,
    "source": "https://www.induction.ai/docs/benchmarks/tau3",
    "read_on": "2026-11-01"
   },
   {
    "effort": "xhigh",
    "cost_change_pct": -60,
    "usd_per_run": {
     "direct": 130,
     "proxy": 52
    },
    "pass_pct": {
     "direct": 40.0,
     "proxy": 39.7
    },
    "tier_match": false,
    "source": "https://www.induction.ai/docs/benchmarks/tau3",
    "read_on": "2026-11-01"
   },
   {
    "effort": "max",
    "cost_change_pct": -67,
    "usd_per_run": {
     "direct": 225,
     "proxy": 74
    },
    "pass_pct": {
     "direct": 43.8,
     "proxy": 47.1
    },
    "tier_match": false,
    "source": "https://www.induction.ai/docs/benchmarks/tau3",
    "read_on": "2026-11-01"
   }
  ],
  "publishes": {
   "trials": false,
   "raw_data": false,
   "plan": false,
   "rerun_path": false
  }
 },
 "proof": {
  "plan_url": "/benchmarks/tau3/confirm-1101/PLAN.md",
  "plan_sha": "0442a3a16c08217e99655b97f78825efa6a502db",
  "plan_pushed_at": "2026-11-01T12:42:42Z",
  "first_paid_call_at": "2026-11-01T12:45:48Z",
  "results_url": "/benchmarks/tau3/confirm-1101/RESULTS.md",
  "results_sha": "1a547af6d898f0caa92c81898dc71ff5b502ae13",
  "raw_release_url": "https://github.com/mksglu/cm-ci-lab/releases/tag/tau3-confirm-1101",
  "sha256sums_url": "https://github.com/mksglu/cm-ci-lab/releases/download/tau3-confirm-1101/SHA256SUMS.txt",
  "raw_files": [
   {
    "name": "direct-s300.json.gz",
    "url": "https://github.com/mksglu/cm-ci-lab/releases/download/tau3-confirm-1101/direct-s300.json.gz",
    "bytes": 10619097,
    "sha256": "17cb530a11fb91a60670c381491c137386e7dd38e1aadeb994f4a2f96fc495f1"
   },
   {
    "name": "direct-s301.json.gz",
    "url": "https://github.com/mksglu/cm-ci-lab/releases/download/tau3-confirm-1101/direct-s301.json.gz",
    "bytes": 7503723,
    "sha256": "36e543c16f864f308a147f1c86293b96596b9da27e564ece447863cb69e5febf"
   },
   {
    "name": "gateway-s300.json.gz",
    "url": "https://github.com/mksglu/cm-ci-lab/releases/download/tau3-confirm-1101/gateway-s300.json.gz",
    "bytes": 11539784,
    "sha256": "b4c9a7d6663f09d1d450c898faa60bf2dbab53f97382f30ad6c53b9da0da8c36"
   },
   {
    "name": "gateway-s301.json.gz",
    "url": "https://github.com/mksglu/cm-ci-lab/releases/download/tau3-confirm-1101/gateway-s301.json.gz",
    "bytes": 11709294,
    "sha256": "c2e4fe8eaa742c0e5f98b6c03d32222aadffacd7d238c051c542368e150bc3c2"
   }
  ]
 }
}