{
 "schema": "ci-bench/1",
 "title": "Headless CI in GitHub Actions: Claude Code direct against Claude Code through Context Mode",
 "set": "night-1101-r2",
 "date": "2026-11-01",
 "status": "first measured round",
 "model": "claude-sonnet-5-5",
 "model_name": "Sonnet 5.5",
 "client": "Claude Code headless (claude -p)",
 "method": [
  "Claude Code runs headless (claude -p) in a GitHub Actions job on real tasks from popular repositories.",
  "Each task runs twice, back to back: direct to Anthropic with ENABLE_TOOL_SEARCH, and through the gateway with the env lines npx @context-mode/cli ci init writes.",
  "Same agent, same prompt, same model in both arms. The live gateway.",
  "Warm runs only: runs 2 to 7 of each sequence, 2 to 3 minutes apart, as real repos rerun a job. The first run (empty cache) is left out.",
  "Cost per run at list price. Each task gives a cost ratio, gateway / direct. A row's saving is 1 minus the geometric mean of its tasks' ratios (the round doc: -(exp(mean log-ratio) - 1)), so it can differ from the ratio of the two mean costs.",
  "95% range: bootstrap over tasks, 10,000 resamples, seed 1.",
  "Publish rule: a row is published only when its whole 95% range is above 0 and both arms did the same work."
 ],
 "proof": {
  "round": "https://github.com/mksglu/cm-ci-lab/blob/main/docs/ci-night-1101-r2.md",
  "raw": "https://github.com/mksglu/cm-ci-lab/blob/main/lab/ci-night/results-r2.json",
  "raw_copy": "/benchmarks/ci/results-r2.json",
  "plan": "https://github.com/mksglu/cm-ci-lab/blob/main/docs/plan-ci-1031.md",
  "plan_written": "2026-10-31T19:27Z",
  "first_run": "2026-11-01T04:22Z",
  "reproduce": "https://github.com/mksglu/cm-ci-lab/blob/main/reproduce/README.md",
  "template": "https://github.com/mksglu/cm-ci-lab/generate",
  "lab": "https://github.com/mksglu/cm-ci-lab"
 },
 "rows": [
  {
   "id": "ci-fix",
   "lane": "triage",
   "label": "CI-failure fix",
   "model": "claude-sonnet-5-5",
   "saving_pct": 33.9,
   "ci_pct": [
    25.1,
    43.6
   ],
   "n": 6,
   "cheaper": 6,
   "direct": 0.0442,
   "gateway": 0.0281,
   "per_1000": 16,
   "net_after_fee": 0.0111,
   "quality": {
    "key": "passed",
    "direct": "7/7",
    "gateway": "7/7",
    "text": "tests pass 7/7 / 7/7"
   },
   "workflow": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/triage.yml",
   "spec": "https://github.com/mksglu/cm-ci-lab/blob/main/reproduce/tasks/triage.json",
   "runs": {
    "s1-direct-ts": 41272848908,
    "s1-gateway": 41272851668,
    "s2-direct-ts": 41273014272,
    "s2-gateway": 41273016796,
    "s3-direct-ts": 41273177239,
    "s3-gateway": 41273179659,
    "s4-gateway": 41273323912,
    "s4-direct-ts": 41273327261,
    "s5-gateway": 41273493111,
    "s5-direct-ts": 41273495925,
    "s6-gateway": 41273654047,
    "s6-direct-ts": 41273657483,
    "s7-direct-ts": 41273821897,
    "s7-gateway": 41273824069
   }
  },
  {
   "id": "triage",
   "lane": "labels",
   "label": "Issue triage and labels",
   "model": "claude-sonnet-5-5",
   "saving_pct": 29.6,
   "ci_pct": [
    21.1,
    38.8
   ],
   "n": 6,
   "cheaper": 6,
   "direct": 0.0262,
   "gateway": 0.0183,
   "per_1000": 8,
   "net_after_fee": 0.006,
   "quality": {
    "key": "f1",
    "direct": "5/7",
    "gateway": "4/7",
    "text": "label F1 pass 5/7 / 4/7"
   },
   "workflow": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/labels.yml",
   "spec": "https://github.com/mksglu/cm-ci-lab/blob/main/reproduce/tasks/labels.json",
   "runs": {
    "s1-direct-ts": 41273878757,
    "s1-gateway": 41273880905,
    "s2-gateway": 41274031210,
    "s2-direct-ts": 41274033739,
    "s3-direct-ts": 41274185701,
    "s3-gateway": 41274188039,
    "s4-gateway": 41274346638,
    "s4-direct-ts": 41274349178,
    "s5-direct-ts": 41274499909,
    "s5-gateway": 41274502078,
    "s6-direct-ts": 41274650649,
    "s6-gateway": 41274653016,
    "s7-gateway": 41274800791,
    "s7-direct-ts": 41274802990
   }
  },
  {
   "id": "release-notes",
   "lane": "release",
   "label": "Release notes",
   "model": "claude-sonnet-5-5",
   "saving_pct": 25.7,
   "ci_pct": [
    7.2,
    42.7
   ],
   "n": 5,
   "cheaper": 4,
   "direct": 0.1333,
   "gateway": 0.0987,
   "per_1000": 35,
   "net_after_fee": 0.0287,
   "quality": {
    "key": null,
    "direct": null,
    "gateway": null,
    "text": "not scored"
   },
   "workflow": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/release.yml",
   "spec": "https://github.com/mksglu/cm-ci-lab/blob/main/reproduce/tasks/release.json",
   "runs": {
    "s1-direct-ts": 41273903809,
    "s1-gateway": 41273906184,
    "s2-gateway": 41274057959,
    "s2-direct-ts": 41274060416,
    "s3-direct-ts": 41274212160,
    "s3-gateway": 41274214653,
    "s4-direct-ts": 41274373942,
    "s4-gateway": 41274376090,
    "s5-gateway": 41274525807,
    "s5-direct-ts": 41274527888,
    "s6-direct-ts": 41274677631,
    "s6-gateway": 41274679730
   }
  },
  {
   "id": "mention-fix",
   "lane": "mention",
   "label": "@claude fix (issue to PR)",
   "model": "claude-sonnet-5-5",
   "saving_pct": 23.7,
   "ci_pct": [
    11.4,
    35.7
   ],
   "n": 6,
   "cheaper": 6,
   "direct": 0.0902,
   "gateway": 0.0736,
   "per_1000": 17,
   "net_after_fee": 0.008,
   "quality": {
    "key": "passed",
    "direct": "5/7",
    "gateway": "5/7",
    "text": "tests pass 5/7 / 5/7"
   },
   "workflow": "https://github.com/mksglu/cm-ci-lab/blob/main/.github/workflows/mention.yml",
   "spec": "https://github.com/mksglu/cm-ci-lab/blob/main/reproduce/tasks/mention.json",
   "runs": {
    "s1-direct-ts": 41272846725,
    "s1-gateway": 41272849393,
    "s2-gateway": 41273011888,
    "s2-direct-ts": 41273014767,
    "s3-gateway": 41273175293,
    "s3-direct-ts": 41273177494,
    "s4-gateway": 41273321179,
    "s4-direct-ts": 41273324007,
    "s5-direct-ts": 41273491335,
    "s5-gateway": 41273493852,
    "s6-gateway": 41273651392,
    "s6-direct-ts": 41273654276,
    "s7-gateway": 41273819704,
    "s7-direct-ts": 41273821862
   }
  }
 ],
 "left_out": [
  {
   "id": "review-action",
   "lane": "review-action",
   "label": "PR review (official claude-code-action)",
   "model": "claude-sonnet-5-5",
   "saving_pct": 10.5,
   "ci_pct": [
    -5.4,
    26.8
   ],
   "n": 6,
   "cheaper": 3,
   "reason": "The 95% range crosses 0, so the row is left out."
  },
  {
   "id": "pr-review",
   "lane": "pr-review",
   "label": "PR review (community template)",
   "model": "claude-sonnet-5-5",
   "saving_pct": 7,
   "ci_pct": [
    -26,
    29
   ],
   "n": 6,
   "cheaper": 4,
   "reason": "The 95% range crosses 0, so the row is left out."
  },
  {
   "id": "dedupe",
   "lane": "dedupe",
   "label": "Issue dedupe",
   "model": "claude-sonnet-5-5",
   "saving_pct": 52.5,
   "ci_pct": [
    24.4,
    71.4
   ],
   "n": 6,
   "cheaper": 6,
   "reason": "The gateway arm made far fewer tool calls (Bash 51 / Agent 25 against Bash 125 / Agent 46), so the two arms did not do the same job and the row is left out."
  }
 ]
}