[{"data":1,"prerenderedAt":1141},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Ffirst":3},{"id":4,"title":5,"body":6,"description":1134,"extension":1135,"meta":1136,"navigation":248,"path":1137,"seo":1138,"stem":1139,"__hash__":1140},"docs\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Ffirst.md","Your First Benchmark",{"type":7,"value":8,"toc":1125},"minimark",[9,13,17,54,57,62,68,191,194,433,440,444,504,515,519,736,743,746,762,766,988,1005,1009,1032,1068,1071,1087,1090,1094,1097,1100,1104,1113,1121],[10,11,5],"h1",{"id":12},"your-first-benchmark",[14,15,16],"p",{},"We are going to measure something concrete: given a genuine bug and a failing test suite,\nwhich model fixes it — and does it touch anything it should not have?",[18,19,24],"pre",{"className":20,"code":21,"language":22,"meta":23,"style":23},"language-bash shiki shiki-themes dark-plus","axon bench init parser-bench\ncd parser-bench\n","bash","",[25,26,27,46],"code",{"__ignoreMap":23},[28,29,32,36,40,43],"span",{"class":30,"line":31},"line",1,[28,33,35],{"class":34},"sCudf","axon",[28,37,39],{"class":38},"sKc5r"," bench",[28,41,42],{"class":38}," init",[28,44,45],{"class":38}," parser-bench\n",[28,47,49,52],{"class":30,"line":48},2,[28,50,51],{"class":34},"cd",[28,53,45],{"class":38},[14,55,56],{},"That scaffolds the whole structure. Four things to fill in.",[58,59,61],"h2",{"id":60},"_1-build-the-world","1. Build the world",[14,63,64,67],{},[25,65,66],{},"workspace\u002F"," is what the agent will see. Put a small project in it with a real defect:",[18,69,73],{"className":70,"code":71,"language":72,"meta":23,"style":23},"language-ts shiki shiki-themes dark-plus","\u002F\u002F workspace\u002Fsrc\u002Fparser.ts\nexport function parseLine(line: string): { key: string; value: string } {\n    const [key, value] = line.split(\"=\")\n    return { key, value }\n}\n","ts",[25,74,75,81,131,167,185],{"__ignoreMap":23},[28,76,77],{"class":30,"line":31},[28,78,80],{"class":79},"sOLPB","\u002F\u002F workspace\u002Fsrc\u002Fparser.ts\n",[28,82,83,87,91,94,98,101,104,108,111,114,116,118,121,124,126,128],{"class":30,"line":48},[28,84,86],{"class":85},"sYbnZ","export",[28,88,90],{"class":89},"scz_3"," function",[28,92,93],{"class":34}," parseLine",[28,95,97],{"class":96},"sTNBD","(",[28,99,30],{"class":100},"s3F5K",[28,102,103],{"class":96},": ",[28,105,107],{"class":106},"sNl3T","string",[28,109,110],{"class":96},"): { ",[28,112,113],{"class":100},"key",[28,115,103],{"class":96},[28,117,107],{"class":106},[28,119,120],{"class":96},"; ",[28,122,123],{"class":100},"value",[28,125,103],{"class":96},[28,127,107],{"class":106},[28,129,130],{"class":96}," } {\n",[28,132,134,137,140,143,146,148,151,153,156,159,161,164],{"class":30,"line":133},3,[28,135,136],{"class":89},"    const",[28,138,139],{"class":96}," [",[28,141,113],{"class":142},"s9McN",[28,144,145],{"class":96},", ",[28,147,123],{"class":142},[28,149,150],{"class":96},"] = ",[28,152,30],{"class":100},[28,154,155],{"class":96},".",[28,157,158],{"class":34},"split",[28,160,97],{"class":96},[28,162,163],{"class":38},"\"=\"",[28,165,166],{"class":96},")\n",[28,168,170,173,176,178,180,182],{"class":30,"line":169},4,[28,171,172],{"class":85},"    return",[28,174,175],{"class":96}," { ",[28,177,113],{"class":100},[28,179,145],{"class":96},[28,181,123],{"class":100},[28,183,184],{"class":96}," }\n",[28,186,188],{"class":30,"line":187},5,[28,189,190],{"class":96},"}\n",[14,192,193],{},"And the suite that catches it:",[18,195,197],{"className":70,"code":196,"language":72,"meta":23,"style":23},"\u002F\u002F workspace\u002Ftests\u002Fparser.test.ts\nimport { expect, it } from \"bun:test\"\nimport { parseLine } from \"..\u002Fsrc\u002Fparser\"\n\nit(\"parses a simple pair\", () => {\n    expect(parseLine(\"a=1\")).toEqual({ key: \"a\", value: \"1\" })\n})\n\nit(\"keeps '=' inside the value\", () => {\n    expect(parseLine(\"url=http:\u002F\u002Fx\u002F?a=1\")).toEqual({ key: \"url\", value: \"http:\u002F\u002Fx\u002F?a=1\" })\n})\n\nit(\"trims surrounding whitespace\", () => {\n    expect(parseLine(\"  a = 1  \")).toEqual({ key: \"a\", value: \"1\" })\n})\n",[25,198,199,204,228,244,250,268,309,315,320,336,370,375,380,396,428],{"__ignoreMap":23},[28,200,201],{"class":30,"line":31},[28,202,203],{"class":79},"\u002F\u002F workspace\u002Ftests\u002Fparser.test.ts\n",[28,205,206,209,211,214,216,219,222,225],{"class":30,"line":48},[28,207,208],{"class":85},"import",[28,210,175],{"class":96},[28,212,213],{"class":100},"expect",[28,215,145],{"class":96},[28,217,218],{"class":100},"it",[28,220,221],{"class":96}," } ",[28,223,224],{"class":85},"from",[28,226,227],{"class":38}," \"bun:test\"\n",[28,229,230,232,234,237,239,241],{"class":30,"line":133},[28,231,208],{"class":85},[28,233,175],{"class":96},[28,235,236],{"class":100},"parseLine",[28,238,221],{"class":96},[28,240,224],{"class":85},[28,242,243],{"class":38}," \"..\u002Fsrc\u002Fparser\"\n",[28,245,246],{"class":30,"line":169},[28,247,249],{"emptyLinePlaceholder":248},true,"\n",[28,251,252,254,256,259,262,265],{"class":30,"line":187},[28,253,218],{"class":34},[28,255,97],{"class":96},[28,257,258],{"class":38},"\"parses a simple pair\"",[28,260,261],{"class":96},", () ",[28,263,264],{"class":89},"=>",[28,266,267],{"class":96}," {\n",[28,269,271,274,276,278,280,283,286,289,292,295,298,300,303,306],{"class":30,"line":270},6,[28,272,273],{"class":34},"    expect",[28,275,97],{"class":96},[28,277,236],{"class":34},[28,279,97],{"class":96},[28,281,282],{"class":38},"\"a=1\"",[28,284,285],{"class":96},")).",[28,287,288],{"class":34},"toEqual",[28,290,291],{"class":96},"({ ",[28,293,294],{"class":100},"key:",[28,296,297],{"class":38}," \"a\"",[28,299,145],{"class":96},[28,301,302],{"class":100},"value:",[28,304,305],{"class":38}," \"1\"",[28,307,308],{"class":96}," })\n",[28,310,312],{"class":30,"line":311},7,[28,313,314],{"class":96},"})\n",[28,316,318],{"class":30,"line":317},8,[28,319,249],{"emptyLinePlaceholder":248},[28,321,323,325,327,330,332,334],{"class":30,"line":322},9,[28,324,218],{"class":34},[28,326,97],{"class":96},[28,328,329],{"class":38},"\"keeps '=' inside the value\"",[28,331,261],{"class":96},[28,333,264],{"class":89},[28,335,267],{"class":96},[28,337,339,341,343,345,347,350,352,354,356,358,361,363,365,368],{"class":30,"line":338},10,[28,340,273],{"class":34},[28,342,97],{"class":96},[28,344,236],{"class":34},[28,346,97],{"class":96},[28,348,349],{"class":38},"\"url=http:\u002F\u002Fx\u002F?a=1\"",[28,351,285],{"class":96},[28,353,288],{"class":34},[28,355,291],{"class":96},[28,357,294],{"class":100},[28,359,360],{"class":38}," \"url\"",[28,362,145],{"class":96},[28,364,302],{"class":100},[28,366,367],{"class":38}," \"http:\u002F\u002Fx\u002F?a=1\"",[28,369,308],{"class":96},[28,371,373],{"class":30,"line":372},11,[28,374,314],{"class":96},[28,376,378],{"class":30,"line":377},12,[28,379,249],{"emptyLinePlaceholder":248},[28,381,383,385,387,390,392,394],{"class":30,"line":382},13,[28,384,218],{"class":34},[28,386,97],{"class":96},[28,388,389],{"class":38},"\"trims surrounding whitespace\"",[28,391,261],{"class":96},[28,393,264],{"class":89},[28,395,267],{"class":96},[28,397,399,401,403,405,407,410,412,414,416,418,420,422,424,426],{"class":30,"line":398},14,[28,400,273],{"class":34},[28,402,97],{"class":96},[28,404,236],{"class":34},[28,406,97],{"class":96},[28,408,409],{"class":38},"\"  a = 1  \"",[28,411,285],{"class":96},[28,413,288],{"class":34},[28,415,291],{"class":96},[28,417,294],{"class":100},[28,419,297],{"class":38},[28,421,145],{"class":96},[28,423,302],{"class":100},[28,425,305],{"class":38},[28,427,308],{"class":96},[28,429,431],{"class":30,"line":430},15,[28,432,314],{"class":96},[14,434,435,436,439],{},"One passes, two fail. ",[25,437,438],{},"split(\"=\")"," drops everything after the second separator and nothing\ntrims. A real bug with a real test suite proving it — which is what makes the measurement\nmean something.",[58,441,443],{"id":442},"_2-declare-what-you-are-measuring","2. Declare what you are measuring",[18,445,447],{"className":70,"code":446,"language":72,"meta":23,"style":23},"type Schema = {\n    \u002F** Has the agent resolved the bug? *\u002F\n    resolved: boolean\n\n    \u002F** Files edited beyond the target. @objective minimize *\u002F\n    collateral: number\n}\n",[25,448,449,460,465,475,479,490,500],{"__ignoreMap":23},[28,450,451,454,457],{"class":30,"line":31},[28,452,453],{"class":89},"type",[28,455,456],{"class":106}," Schema",[28,458,459],{"class":96}," = {\n",[28,461,462],{"class":30,"line":48},[28,463,464],{"class":79},"    \u002F** Has the agent resolved the bug? *\u002F\n",[28,466,467,470,472],{"class":30,"line":133},[28,468,469],{"class":100},"    resolved",[28,471,103],{"class":96},[28,473,474],{"class":106},"boolean\n",[28,476,477],{"class":30,"line":169},[28,478,249],{"emptyLinePlaceholder":248},[28,480,481,484,487],{"class":30,"line":187},[28,482,483],{"class":79},"    \u002F** Files edited beyond the target. ",[28,485,486],{"class":89},"@objective",[28,488,489],{"class":79}," minimize *\u002F\n",[28,491,492,495,497],{"class":30,"line":270},[28,493,494],{"class":100},"    collateral",[28,496,103],{"class":96},[28,498,499],{"class":106},"number\n",[28,501,502],{"class":30,"line":311},[28,503,190],{"class":96},[14,505,506,507,510,511,514],{},"Two measurements. ",[25,508,509],{},"resolved"," needs no objective — more fixed bugs being better is obvious.\n",[25,512,513],{},"collateral"," does, because fewer touched files being better is not something a chart can\ninfer.",[58,516,518],{"id":517},"_3-declare-what-varies","3. Declare what varies",[18,520,522],{"className":70,"code":521,"language":72,"meta":23,"style":23},"import { OpenRouter } from \"@arcforge\u002Fengines\"\n\nexport default defineBench\u003CSchema>({\n    description: \"Can the agent fix a real bug without collateral damage?\",\n\n    workspace: { source: \".\u002Fworkspace\", retain: \"failed\" },\n\n    matrix: {\n        model: [\n            OpenRouter({ model: \"anthropic\u002Fclaude-sonnet-4.6\" }),\n            OpenRouter({ model: \"openai\u002Fgpt-5\" }),\n        ],\n    },\n\n    trials: 3,\n\n    async setup() {\n        await Bun.$`bun install`.cwd(bench.workspace)\n    },\n})\n",[25,523,524,540,544,563,574,578,602,606,613,621,637,650,655,660,664,675,680,692,726,731],{"__ignoreMap":23},[28,525,526,528,530,533,535,537],{"class":30,"line":31},[28,527,208],{"class":85},[28,529,175],{"class":96},[28,531,532],{"class":100},"OpenRouter",[28,534,221],{"class":96},[28,536,224],{"class":85},[28,538,539],{"class":38}," \"@arcforge\u002Fengines\"\n",[28,541,542],{"class":30,"line":48},[28,543,249],{"emptyLinePlaceholder":248},[28,545,546,548,551,554,557,560],{"class":30,"line":133},[28,547,86],{"class":85},[28,549,550],{"class":85}," default",[28,552,553],{"class":34}," defineBench",[28,555,556],{"class":96},"\u003C",[28,558,559],{"class":106},"Schema",[28,561,562],{"class":96},">({\n",[28,564,565,568,571],{"class":30,"line":169},[28,566,567],{"class":100},"    description:",[28,569,570],{"class":38}," \"Can the agent fix a real bug without collateral damage?\"",[28,572,573],{"class":96},",\n",[28,575,576],{"class":30,"line":187},[28,577,249],{"emptyLinePlaceholder":248},[28,579,580,583,585,588,591,593,596,599],{"class":30,"line":270},[28,581,582],{"class":100},"    workspace:",[28,584,175],{"class":96},[28,586,587],{"class":100},"source:",[28,589,590],{"class":38}," \".\u002Fworkspace\"",[28,592,145],{"class":96},[28,594,595],{"class":100},"retain:",[28,597,598],{"class":38}," \"failed\"",[28,600,601],{"class":96}," },\n",[28,603,604],{"class":30,"line":311},[28,605,249],{"emptyLinePlaceholder":248},[28,607,608,611],{"class":30,"line":317},[28,609,610],{"class":100},"    matrix:",[28,612,267],{"class":96},[28,614,615,618],{"class":30,"line":322},[28,616,617],{"class":100},"        model:",[28,619,620],{"class":96}," [\n",[28,622,623,626,628,631,634],{"class":30,"line":338},[28,624,625],{"class":34},"            OpenRouter",[28,627,291],{"class":96},[28,629,630],{"class":100},"model:",[28,632,633],{"class":38}," \"anthropic\u002Fclaude-sonnet-4.6\"",[28,635,636],{"class":96}," }),\n",[28,638,639,641,643,645,648],{"class":30,"line":372},[28,640,625],{"class":34},[28,642,291],{"class":96},[28,644,630],{"class":100},[28,646,647],{"class":38}," \"openai\u002Fgpt-5\"",[28,649,636],{"class":96},[28,651,652],{"class":30,"line":377},[28,653,654],{"class":96},"        ],\n",[28,656,657],{"class":30,"line":382},[28,658,659],{"class":96},"    },\n",[28,661,662],{"class":30,"line":398},[28,663,249],{"emptyLinePlaceholder":248},[28,665,666,669,673],{"class":30,"line":430},[28,667,668],{"class":100},"    trials:",[28,670,672],{"class":671},"sDGX8"," 3",[28,674,573],{"class":96},[28,676,678],{"class":30,"line":677},16,[28,679,249],{"emptyLinePlaceholder":248},[28,681,683,686,689],{"class":30,"line":682},17,[28,684,685],{"class":89},"    async",[28,687,688],{"class":34}," setup",[28,690,691],{"class":96},"() {\n",[28,693,695,698,701,703,706,709,711,714,716,719,721,724],{"class":30,"line":694},18,[28,696,697],{"class":85},"        await",[28,699,700],{"class":100}," Bun",[28,702,155],{"class":96},[28,704,705],{"class":34},"$",[28,707,708],{"class":38},"`bun install`",[28,710,155],{"class":96},[28,712,713],{"class":34},"cwd",[28,715,97],{"class":96},[28,717,718],{"class":100},"bench",[28,720,155],{"class":96},[28,722,723],{"class":100},"workspace",[28,725,166],{"class":96},[28,727,729],{"class":30,"line":728},19,[28,730,659],{"class":96},[28,732,734],{"class":30,"line":733},20,[28,735,314],{"class":96},[14,737,738,739,742],{},"Two models, three trials: six runs. ",[25,740,741],{},"setup()"," installs the workspace's dependencies after\nthe copy and before the agent boots, so the test suite is runnable when the agent arrives.",[14,744,745],{},"Add the engines package so the import resolves:",[18,747,749],{"className":20,"code":748,"language":22,"meta":23,"style":23},"bun add @arcforge\u002Fengines\n",[25,750,751],{"__ignoreMap":23},[28,752,753,756,759],{"class":30,"line":31},[28,754,755],{"class":34},"bun",[28,757,758],{"class":38}," add",[28,760,761],{"class":38}," @arcforge\u002Fengines\n",[58,763,765],{"id":764},"_4-write-the-scenario","4. Write the scenario",[18,767,769],{"className":70,"code":768,"language":72,"meta":23,"style":23},"\u002F\u002F tests\u002Ffix-bug.bench.ts\nit(\"resolves the failing test\", async ({ workspace }) => {\n    expect(workspace).toHaveFile(\"src\u002Fparser.ts\")\n\n    const { axon } = await Axon()\n    await axon.request(\"Two tests in tests\u002Fparser.test.ts are failing. Fix src\u002Fparser.ts so they pass.\")\n\n    observe(\"resolved\", await workspace.tests.pass())\n    observe(\"collateral\", (await workspace.changed()).filter(f => f !== \"src\u002Fparser.ts\").length)\n\n    await bench.attach(\"diff\", await workspace.diff())\n})\n",[25,770,771,776,802,823,827,847,867,871,901,951,955,984],{"__ignoreMap":23},[28,772,773],{"class":30,"line":31},[28,774,775],{"class":79},"\u002F\u002F tests\u002Ffix-bug.bench.ts\n",[28,777,778,780,782,785,787,790,793,795,798,800],{"class":30,"line":48},[28,779,218],{"class":34},[28,781,97],{"class":96},[28,783,784],{"class":38},"\"resolves the failing test\"",[28,786,145],{"class":96},[28,788,789],{"class":89},"async",[28,791,792],{"class":96}," ({ ",[28,794,723],{"class":100},[28,796,797],{"class":96}," }) ",[28,799,264],{"class":89},[28,801,267],{"class":96},[28,803,804,806,808,810,813,816,818,821],{"class":30,"line":133},[28,805,273],{"class":34},[28,807,97],{"class":96},[28,809,723],{"class":100},[28,811,812],{"class":96},").",[28,814,815],{"class":34},"toHaveFile",[28,817,97],{"class":96},[28,819,820],{"class":38},"\"src\u002Fparser.ts\"",[28,822,166],{"class":96},[28,824,825],{"class":30,"line":169},[28,826,249],{"emptyLinePlaceholder":248},[28,828,829,831,833,835,838,841,844],{"class":30,"line":187},[28,830,136],{"class":89},[28,832,175],{"class":96},[28,834,35],{"class":142},[28,836,837],{"class":96}," } = ",[28,839,840],{"class":85},"await",[28,842,843],{"class":34}," Axon",[28,845,846],{"class":96},"()\n",[28,848,849,852,855,857,860,862,865],{"class":30,"line":270},[28,850,851],{"class":85},"    await",[28,853,854],{"class":100}," axon",[28,856,155],{"class":96},[28,858,859],{"class":34},"request",[28,861,97],{"class":96},[28,863,864],{"class":38},"\"Two tests in tests\u002Fparser.test.ts are failing. Fix src\u002Fparser.ts so they pass.\"",[28,866,166],{"class":96},[28,868,869],{"class":30,"line":311},[28,870,249],{"emptyLinePlaceholder":248},[28,872,873,876,878,881,883,885,888,890,893,895,898],{"class":30,"line":317},[28,874,875],{"class":34},"    observe",[28,877,97],{"class":96},[28,879,880],{"class":38},"\"resolved\"",[28,882,145],{"class":96},[28,884,840],{"class":85},[28,886,887],{"class":100}," workspace",[28,889,155],{"class":96},[28,891,892],{"class":100},"tests",[28,894,155],{"class":96},[28,896,897],{"class":34},"pass",[28,899,900],{"class":96},"())\n",[28,902,903,905,907,910,913,915,917,919,922,925,928,930,933,936,939,942,944,946,949],{"class":30,"line":322},[28,904,875],{"class":34},[28,906,97],{"class":96},[28,908,909],{"class":38},"\"collateral\"",[28,911,912],{"class":96},", (",[28,914,840],{"class":85},[28,916,887],{"class":100},[28,918,155],{"class":96},[28,920,921],{"class":34},"changed",[28,923,924],{"class":96},"()).",[28,926,927],{"class":34},"filter",[28,929,97],{"class":96},[28,931,932],{"class":100},"f",[28,934,935],{"class":89}," =>",[28,937,938],{"class":100}," f",[28,940,941],{"class":96}," !== ",[28,943,820],{"class":38},[28,945,812],{"class":96},[28,947,948],{"class":100},"length",[28,950,166],{"class":96},[28,952,953],{"class":30,"line":338},[28,954,249],{"emptyLinePlaceholder":248},[28,956,957,959,961,963,966,968,971,973,975,977,979,982],{"class":30,"line":372},[28,958,851],{"class":85},[28,960,39],{"class":100},[28,962,155],{"class":96},[28,964,965],{"class":34},"attach",[28,967,97],{"class":96},[28,969,970],{"class":38},"\"diff\"",[28,972,145],{"class":96},[28,974,840],{"class":85},[28,976,887],{"class":100},[28,978,155],{"class":96},[28,980,981],{"class":34},"diff",[28,983,900],{"class":96},[28,985,986],{"class":30,"line":377},[28,987,314],{"class":96},[14,989,990,991,993,994,997,998,1001,1002,1004],{},"The ",[25,992,213],{}," guards the experiment — if ",[25,995,996],{},"src\u002Fparser.ts"," is missing, the world is wrong and\nthis trial measures nothing. The two ",[25,999,1000],{},"observe"," calls record what happened, whatever it was.\nThe ",[25,1003,965],{}," keeps the actual patch, because when a cell scores badly the diff is what tells\nyou why.",[58,1006,1008],{"id":1007},"run-it","Run it",[18,1010,1012],{"className":20,"code":1011,"language":22,"meta":23,"style":23},"axon bench prepare\naxon bench run\n",[25,1013,1014,1023],{"__ignoreMap":23},[28,1015,1016,1018,1020],{"class":30,"line":31},[28,1017,35],{"class":34},[28,1019,39],{"class":38},[28,1021,1022],{"class":38}," prepare\n",[28,1024,1025,1027,1029],{"class":30,"line":48},[28,1026,35],{"class":34},[28,1028,39],{"class":38},[28,1030,1031],{"class":38}," run\n",[18,1033,1035],{"className":20,"code":1034,"language":22,"meta":23,"style":23},"Benchmark run 01JQZX8K2M4N6P8R\n  4 passed, 2 failed, 12 observations\n",[25,1036,1037,1048],{"__ignoreMap":23},[28,1038,1039,1042,1045],{"class":30,"line":31},[28,1040,1041],{"class":34},"Benchmark",[28,1043,1044],{"class":38}," run",[28,1046,1047],{"class":38}," 01JQZX8K2M4N6P8R\n",[28,1049,1050,1053,1056,1059,1062,1065],{"class":30,"line":48},[28,1051,1052],{"class":34},"  4",[28,1054,1055],{"class":38}," passed,",[28,1057,1058],{"class":671}," 2",[28,1060,1061],{"class":38}," failed,",[28,1063,1064],{"class":671}," 12",[28,1066,1067],{"class":38}," observations\n",[14,1069,1070],{},"Twelve observations from six runs: two measurements each. The full record:",[18,1072,1074],{"className":20,"code":1073,"language":22,"meta":23,"style":23},"axon bench result 01JQZX8K2M4N6P8R\n",[25,1075,1076],{"__ignoreMap":23},[28,1077,1078,1080,1082,1085],{"class":30,"line":31},[28,1079,35],{"class":34},[28,1081,39],{"class":38},[28,1083,1084],{"class":38}," result",[28,1086,1047],{"class":38},[14,1088,1089],{},"Every trial, its physics, its session, its observations, and the manifest pinning exactly\nwhich model produced which number.",[58,1091,1093],{"id":1092},"what-you-did-not-have-to-write","What you did not have to write",[14,1095,1096],{},"Duration, token counts, cost, engine calls, tool calls, error counts. Which trials\ncompleted and which faulted, and why. The full session for every run. Coverage — whether\nevery measurement you declared actually fired.",[14,1098,1099],{},"None of that is in the config or the test. It falls out of events the runtime already\nemits, which means it is there for every benchmark ever written.",[58,1101,1103],{"id":1102},"where-to-go-next","Where to go next",[14,1105,1106,1107,1112],{},"Cut the cost of iterating to zero with ",[1108,1109,1111],"a",{"href":1110},"\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Fmock","Testing with Mock"," —\nthe same six-run flow, no API keys, no spend.",[14,1114,1115,1116,1120],{},"Then ",[1108,1117,1119],{"href":1118},"\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Fpublishing","Publishing",", so somebody else can run this against\ntheir agent.",[1122,1123,1124],"style",{},"html pre.shiki code .sCudf, html code.shiki .sCudf{--shiki-default:#DCDCAA}html pre.shiki code .sKc5r, html code.shiki .sKc5r{--shiki-default:#CE9178}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sOLPB, html code.shiki .sOLPB{--shiki-default:#6A9955}html pre.shiki code .sYbnZ, html code.shiki .sYbnZ{--shiki-default:#C586C0}html pre.shiki code .scz_3, html code.shiki .scz_3{--shiki-default:#569CD6}html pre.shiki code .sTNBD, html code.shiki .sTNBD{--shiki-default:#D4D4D4}html pre.shiki code .s3F5K, html code.shiki .s3F5K{--shiki-default:#9CDCFE}html pre.shiki code .sNl3T, html code.shiki .sNl3T{--shiki-default:#4EC9B0}html pre.shiki code .s9McN, html code.shiki .s9McN{--shiki-default:#4FC1FF}html pre.shiki code .sDGX8, html code.shiki .sDGX8{--shiki-default:#B5CEA8}",{"title":23,"searchDepth":48,"depth":48,"links":1126},[1127,1128,1129,1130,1131,1132,1133],{"id":60,"depth":48,"text":61},{"id":442,"depth":48,"text":443},{"id":517,"depth":48,"text":518},{"id":764,"depth":48,"text":765},{"id":1007,"depth":48,"text":1008},{"id":1092,"depth":48,"text":1093},{"id":1102,"depth":48,"text":1103},"A real bug, two models, end to end.","md",{},"\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Ffirst",{"title":5,"description":1134},"docs\u002Fv2\u002Fbench\u002Fbuilding\u002Ffirst","rFIJ4XrdtgkasiSPO19jp3YjaYs_e3unknWwWR76Ye4",1785237008155]