{
 "schema": "ci-confirm/1",
 "title": "Headless CI: Claude Code direct against Claude Code through Context Mode",
 "set": "confirm-1101-r2",
 "date": "2026-11-01",
 "status": "pre-registered, audited",
 "client": "Claude Code headless (claude -p)",
 "method": [
  "Claude Code runs headless (claude -p) in a GitHub Actions job on real tasks from popular repositories.",
  "Each task runs twice, once direct to the API and once through the gateway. The two arms use separate Anthropic accounts, so they cannot share a cache. The arm that goes first alternates.",
  "Same agent, same prompt, same model in both arms.",
  "Cost is provider token usage at Anthropic list price. The gateway arm uses the gateway ledger, the higher figure, so a gap works against us.",
  "Saving is 1 minus the geometric mean of gateway cost over direct cost across tasks. The 95% range is a paired bootstrap over tasks (10,000 draws, seed 1).",
  "Bar, fixed before the runs: the whole 95% range above 0, quality not lower, and the same work done (tool-call ratio 0.7 or more)."
 ],
 "rows": [
  {
   "id": "fix-ci-sonnet",
   "use_case": "fix-ci",
   "label": "CI failure fix",
   "agent": "Claude Code",
   "model": "claude-sonnet-5-5",
   "model_name": "Sonnet 5.5",
   "n": 16,
   "saving_pct": 22.8,
   "ci_low": 11.6,
   "ci_high": 32.2,
   "cheaper_pairs": 14,
   "direct_usd": 0.0502,
   "gateway_usd": 0.042,
   "saved_per_1000_runs": 8,
   "quality": {
    "key": "tests",
    "direct": "16/16",
    "gateway": "15/16",
    "text": "Tests pass 16 of 16 direct, 15 of 16 with Context Mode (McNemar p = 1)"
   },
   "workflow_url": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/confirm-claude.yml",
   "tasks_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/tasks-r2/fix-ci.json"
  },
  {
   "id": "fix-mention-sonnet",
   "use_case": "fix-mention",
   "label": "@claude fix (issue to PR)",
   "agent": "Claude Code",
   "model": "claude-sonnet-5-5",
   "model_name": "Sonnet 5.5",
   "n": 19,
   "saving_pct": 21.8,
   "ci_low": 10.3,
   "ci_high": 32.1,
   "cheaper_pairs": 16,
   "direct_usd": 0.1113,
   "gateway_usd": 0.0872,
   "saved_per_1000_runs": 24,
   "quality": {
    "key": "tests",
    "direct": "9/19",
    "gateway": "10/19",
    "text": "Tests pass 9 of 19 direct, 10 of 19 with Context Mode (McNemar p = 1)"
   },
   "workflow_url": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/confirm-claude.yml",
   "tasks_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/tasks-r2/fix-mention.json"
  },
  {
   "id": "labels-sonnet",
   "use_case": "labels",
   "label": "Issue triage and labels",
   "agent": "Claude Code",
   "model": "claude-sonnet-5-5",
   "model_name": "Sonnet 5.5",
   "n": 26,
   "saving_pct": 22.4,
   "ci_low": 15.5,
   "ci_high": 27.7,
   "cheaper_pairs": 24,
   "direct_usd": 0.0281,
   "gateway_usd": 0.0223,
   "saved_per_1000_runs": 6,
   "quality": {
    "key": "f1",
    "direct": "0.499",
    "gateway": "0.478",
    "text": "Label F1 0.499 direct, 0.478 with Context Mode, inside the 0.05 margin"
   },
   "workflow_url": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/confirm-claude.yml",
   "tasks_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/tasks-r2/labels.json"
  },
  {
   "id": "fix-ci-opus",
   "use_case": "fix-ci",
   "label": "CI failure fix",
   "agent": "Claude Code",
   "model": "claude-opus-5-5",
   "model_name": "Opus 5.5",
   "n": 8,
   "saving_pct": 19.7,
   "ci_low": 4.1,
   "ci_high": 31.5,
   "cheaper_pairs": 6,
   "direct_usd": 0.1691,
   "gateway_usd": 0.132,
   "saved_per_1000_runs": 37,
   "quality": {
    "key": "tests",
    "direct": "8/8",
    "gateway": "8/8",
    "text": "Tests pass 8 of 8 direct, 8 of 8 with Context Mode (McNemar p = 1)"
   },
   "workflow_url": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/confirm-claude.yml",
   "tasks_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/tasks-r2/fix-ci.json"
  },
  {
   "id": "fix-mention-opus",
   "use_case": "fix-mention",
   "label": "@claude fix (issue to PR)",
   "agent": "Claude Code",
   "model": "claude-opus-5-5",
   "model_name": "Opus 5.5",
   "n": 8,
   "saving_pct": 21.4,
   "ci_low": 7.1,
   "ci_high": 35.4,
   "cheaper_pairs": 7,
   "direct_usd": 0.335,
   "gateway_usd": 0.2759,
   "saved_per_1000_runs": 59,
   "quality": {
    "key": "tests",
    "direct": "3/8",
    "gateway": "3/8",
    "text": "Tests pass 3 of 8 direct, 3 of 8 with Context Mode (McNemar p = 1)"
   },
   "workflow_url": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/confirm-claude.yml",
   "tasks_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/tasks-r2/fix-mention.json"
  }
 ],
 "proof": {
  "plan_url": "https://github.com/mksglu/cm-ci-lab/blob/a4770f78c0767d21380bc3bf510c32f9a886cbac/confirm-1101/PLAN-r2.md",
  "plan_sha": "a4770f78c0767d21380bc3bf510c32f9a886cbac",
  "plan_pushed_at": "2026-11-01T23:39:27Z",
  "first_paid_run_at": "2026-11-01T23:40:03Z",
  "results_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/RESULTS-r2.md",
  "run_ids_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/run2/run-ids.txt",
  "manifest_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/MANIFEST.sha256",
  "rows_url": "https://github.com/mksglu/cm-ci-lab/blob/main/confirm-1101/run2/final.json",
  "reproduce_url": "https://github.com/mksglu/cm-ci-lab/blob/main/reproduce/README.md",
  "template_url": "https://github.com/mksglu/cm-ci-lab/generate",
  "lab_url": "https://github.com/mksglu/cm-ci-lab"
 },
 "audit": "2026-11-02: 7 of 7 checks passed. The plan was pushed before any paid run, and every result file was rebuilt byte for byte."
}