[
 {
  "kind": "Hard legal research",
  "order": 0,
  "bench": "A generated legislation set: made-up jurisdictions of about 2,830 documents, 5 hard questions (Experiment 5)",
  "axis": "Runs right (%)",
  "note": "Each question has an obvious answer that a later law, amendment or court decision overrides. No published scores exist for this set."
 },
 {
  "kind": "Legal decisions (tax law)",
  "order": 1,
  "bench": "DeonticBench: tax owed, right within $1",
  "axis": "Runs right (%)",
  "note": "Experiment 6. The swarms reached 1 or 2 of the 4 items before the box was torn down."
 },
 {
  "kind": "Decisions under rules",
  "order": 2,
  "bench": "RuleArena: NBA salary-cap rulings",
  "axis": "Runs right (%)",
  "note": "Experiment 6. The swarms reached 1 of the 4 items."
 },
 {
  "kind": "Data analysis",
  "order": 3,
  "bench": "KramaBench: hard data questions",
  "axis": "Runs right (%)",
  "note": "Experiment 6. The swarms reached 1 of the 4 items."
 },
 {
  "kind": "Document retrieval",
  "order": 4,
  "bench": "BRIGHT: nDCG@10, out of 100",
  "axis": "Retrieval score (nDCG@10)",
  "note": "Experiment 6. OpenResearch hit the 40-minute cap reading very large files and gave no answer."
 },
 {
  "kind": "Finding vulnerable files",
  "order": 5,
  "bench": "VLoc: File F1, out of 100",
  "axis": "File F1 (out of 100)",
  "note": "Experiment 6. OpenResearch\u2019s only run was cut off by the teardown, so it has no bar."
 },
 {
  "kind": "Kernel challenge",
  "order": 6,
  "bench": "Anthropic kernel challenge: make a kernel run in fewer cycles",
  "axis": "Times faster than the original kernel",
  "note": "OpenEvolve with GLM agents, steered only by the OpenResearch add-on\u2019s Sonnet decider; cycles were measured only afterwards, so this is not a numeric-evaluator result, and the decider\u2019s Claude usage is not in the cost. Anthropic\u2019s runs started from a much faster version of the kernel, and neither their cost nor most of their times were published."
 }
]