[{"data":1,"prerenderedAt":681},["ShallowReactive",2],{"i-mdi:magnify":3,"i-simple-icons:visualstudiocode":8,"i-mdi:github":11,"i-mdi:discord":13,"i-mdi:youtube":15,"i-mdi:flask-outline":17,"i-vscode-icons:default-folder-opened":19,"i-vscode-icons:default-folder":22,"i-vscode-icons:folder-type-test":24,"i-vscode-icons:file-type-typescript":26,"i-vscode-icons:file-type-json":28,"docs-\u002Fdocs\u002Fv2\u002Fbench\u002Ftests":30},{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":7},0,24,false,"\u003Cpath fill=\"currentColor\" d=\"M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5l-1.5 1.5l-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16A6.5 6.5 0 0 1 3 9.5A6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14S14 12 14 9.5S12 5 9.5 5\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":9,"hidden":10},"\u003Cpath fill=\"currentColor\" d=\"M23.15 2.587L18.21.21a1.49 1.49 0 0 0-1.705.29l-9.46 8.63l-4.12-3.128a1 1 0 0 0-1.276.057L.327 7.261A1 1 0 0 0 .326 8.74L3.899 12L.326 15.26a1 1 0 0 0 .001 1.479L1.65 17.94a1 1 0 0 0 1.276.057l4.12-3.128l9.46 8.63a1.49 1.49 0 0 0 1.704.29l4.942-2.377A1.5 1.5 0 0 0 24 20.06V3.939a1.5 1.5 0 0 0-.85-1.352m-5.146 14.861L10.826 12l7.178-5.448z\"\u002F>",true,{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":12},"\u003Cpath fill=\"currentColor\" d=\"M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5c.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34c-.46-1.16-1.11-1.47-1.11-1.47c-.91-.62.07-.6.07-.6c1 .07 1.53 1.03 1.53 1.03c.87 1.52 2.34 1.07 2.91.83c.09-.65.35-1.09.63-1.34c-2.22-.25-4.55-1.11-4.55-4.92c0-1.11.38-2 1.03-2.71c-.1-.25-.45-1.29.1-2.64c0 0 .84-.27 2.75 1.02c.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02c.55 1.35.2 2.39.1 2.64c.65.71 1.03 1.6 1.03 2.71c0 3.82-2.34 4.66-4.57 4.91c.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":14,"hidden":10},"\u003Cpath fill=\"currentColor\" d=\"m22 24l-5.25-5l.63 2H4.5A2.5 2.5 0 0 1 2 18.5v-15A2.5 2.5 0 0 1 4.5 1h15A2.5 2.5 0 0 1 22 3.5V24M12 6.8c-2.68 0-4.56 1.15-4.56 1.15c1.03-.92 2.83-1.45 2.83-1.45l-.17-.17c-1.69.03-3.22 1.2-3.22 1.2c-1.72 3.59-1.61 6.69-1.61 6.69c1.4 1.81 3.48 1.68 3.48 1.68l.71-.9c-1.25-.27-2.04-1.38-2.04-1.38S9.3 14.9 12 14.9s4.58-1.28 4.58-1.28s-.79 1.11-2.04 1.38l.71.9s2.08.13 3.48-1.68c0 0 .11-3.1-1.61-6.69c0 0-1.53-1.17-3.22-1.2l-.17.17s1.8.53 2.83 1.45c0 0-1.88-1.15-4.56-1.15m-2.07 3.79c.65 0 1.18.57 1.17 1.27c0 .69-.52 1.27-1.17 1.27c-.64 0-1.16-.58-1.16-1.27c0-.7.51-1.27 1.16-1.27m4.17 0c.65 0 1.17.57 1.17 1.27c0 .69-.52 1.27-1.17 1.27c-.64 0-1.16-.58-1.16-1.27c0-.7.51-1.27 1.16-1.27Z\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":16},"\u003Cpath fill=\"currentColor\" d=\"m10 15l5.19-3L10 9zm11.56-7.83c.13.47.22 1.1.28 1.9c.07.8.1 1.49.1 2.09L22 12c0 2.19-.16 3.8-.44 4.83c-.25.9-.83 1.48-1.73 1.73c-.47.13-1.33.22-2.65.28c-1.3.07-2.49.1-3.59.1L12 19c-4.19 0-6.8-.16-7.83-.44c-.9-.25-1.48-.83-1.73-1.73c-.13-.47-.22-1.1-.28-1.9c-.07-.8-.1-1.49-.1-2.09L2 12c0-2.19.16-3.8.44-4.83c.25-.9.83-1.48 1.73-1.73c.47-.13 1.33-.22 2.65-.28c1.3-.07 2.49-.1 3.59-.1L12 5c4.19 0 6.8.16 7.83.44c.9.25 1.48.83 1.73 1.73\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":18},"\u003Cpath fill=\"currentColor\" d=\"M5 19a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1c0-.21-.07-.41-.18-.57L13 8.35V4h-2v4.35L5.18 18.43c-.11.16-.18.36-.18.57m1 3a3 3 0 0 1-3-3c0-.6.18-1.16.5-1.63L9 7.81V6a1 1 0 0 1-1-1V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v1a1 1 0 0 1-1 1v1.81l5.5 9.56c.32.47.5 1.03.5 1.63a3 3 0 0 1-3 3zm7-6l1.34-1.34L16.27 18H7.73l2.66-4.61zm-.5-4a.5.5 0 0 1 .5.5a.5.5 0 0 1-.5.5a.5.5 0 0 1-.5-.5a.5.5 0 0 1 .5-.5\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":21},32,"\u003Cpath fill=\"#dcb67a\" d=\"M27.4 5.5h-9.2l-2.1 4.2H4.3v16.8h25.2v-21Zm0 18.7H6.6V11.8h20.8Zm0-14.5h-8.2l1-2.1h7.1v2.1Z\"\u002F>\u003Cpath fill=\"#dcb67a\" d=\"M25.7 13.7H.5l3.8 12.8h25.2z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":23},"\u003Cpath fill=\"#c09553\" d=\"M27.5 5.5h-9.3l-2.1 4.2H4.4v16.8h25.2v-21Zm0 4.2h-8.2l1.1-2.1h7.1Z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":25},"\u003Cpath fill=\"#b70b0b\" d=\"M27.5 5.5h-9.3l-2.1 4.2H4.4v16.8h25.2v-21Zm0 4.2h-8.2l1.1-2.1h7.1Z\"\u002F>\u003Cpath fill=\"#fac6c5\" d=\"M28.3 27.423c-1.729-.4-2.332-2.482-1.329-4.583a8.95 8.95 0 0 1 2.217-2.779l.314-.26l.009.357a6.7 6.7 0 0 0 .748 2.425a14 14 0 0 1 .625 1.631a3.2 3.2 0 0 1-.073 1.958a2.67 2.67 0 0 1-1.111 1.087a2.8 2.8 0 0 1-1.4.164m.351-3.565c.124-.143.155-.332.164-1l.011-.825l-.2.158a1.68 1.68 0 0 0-.635 1.4c.07.214.286.448.411.446a.5.5 0 0 0 .249-.179m-15.527 7.087a3.85 3.85 0 0 1-3.031-4.563c.254-1.07.2-1.008 5.726-6.787l5.2-5.442l.738.7l.738.7l-3 3.132l-3 3.132l2.508-.054l2.508-.054l1.773-1.85l1.773-1.85l.735.707l.735.707l-5.213 5.43c-4.783 4.982-5.257 5.452-5.743 5.692a3.9 3.9 0 0 1-2.452.393Zm14.662-12.484c-.138-.082-1.546-1.4-3.129-2.921l-2.879-2.772l-.009-.421a.86.86 0 0 1 .194-.673A.99.99 0 0 1 23 11.3a69 69 0 0 1 6.277 6.023a1 1 0 0 1-1.492 1.133Z\"\u002F>\u003Cpath fill=\"#fac6c5\" d=\"M11.578 26.851a2.1 2.1 0 0 0 2.579 2.614c.191-.2 2.407-2.515 7.387-7.737c-2.505.054-2.475-.055-4.98 0c-5.015 5.195-5.114 5.157-4.986 5.123\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":27},"\u003Cpath fill=\"#007acc\" d=\"M23.827 8.243a4.4 4.4 0 0 1 2.223 1.281a6 6 0 0 1 .852 1.143c.011.045-1.534 1.083-2.471 1.662c-.034.023-.169-.124-.322-.35a2.01 2.01 0 0 0-1.67-1c-1.077-.074-1.771.49-1.766 1.433a1.3 1.3 0 0 0 .153.666c.237.49.677.784 2.059 1.383c2.544 1.095 3.636 1.817 4.31 2.843a5.16 5.16 0 0 1 .416 4.333a4.76 4.76 0 0 1-3.932 2.815a11 11 0 0 1-2.708-.028a6.53 6.53 0 0 1-3.616-1.884a6.3 6.3 0 0 1-.926-1.371a3 3 0 0 1 .327-.208c.158-.09.756-.434 1.32-.761l1.024-.6l.214.312a4.8 4.8 0 0 0 1.35 1.292a3.3 3.3 0 0 0 3.458-.175a1.545 1.545 0 0 0 .2-1.974c-.276-.395-.84-.727-2.443-1.422a8.8 8.8 0 0 1-3.349-2.055a4.7 4.7 0 0 1-.976-1.777a7.1 7.1 0 0 1-.062-2.268a4.33 4.33 0 0 1 3.644-3.374a9 9 0 0 1 2.691.084m-8.343 1.483l.011 1.454h-4.63v13.148H7.6V11.183H2.97V9.755a14 14 0 0 1 .04-1.466c.017-.023 2.832-.034 6.245-.028l6.211.017Z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":29},"\u003Cpath fill=\"#f5de19\" d=\"M4.014 14.976a2.5 2.5 0 0 0 1.567-.518a2.38 2.38 0 0 0 .805-1.358a15.3 15.3 0 0 0 .214-2.944q.012-2.085.075-2.747a5.2 5.2 0 0 1 .418-1.686a3 3 0 0 1 .755-1.018A3.05 3.05 0 0 1 9 4.125A6.8 6.8 0 0 1 10.544 4h.7v1.96h-.387a2.34 2.34 0 0 0-1.723.468a3.4 3.4 0 0 0-.425 2.092a36 36 0 0 1-.137 4.133a4.7 4.7 0 0 1-.768 2.06A4.6 4.6 0 0 1 6.1 16a3.8 3.8 0 0 1 1.992 1.754a8.9 8.9 0 0 1 .618 3.865q0 2.435.05 2.9a1.76 1.76 0 0 0 .504 1.181a2.64 2.64 0 0 0 1.592.337h.387V28h-.7a5.7 5.7 0 0 1-1.773-.2a2.97 2.97 0 0 1-1.324-.93a3.35 3.35 0 0 1-.681-1.63a24 24 0 0 1-.165-3.234a16.5 16.5 0 0 0-.214-3.106a2.4 2.4 0 0 0-.805-1.361a2.5 2.5 0 0 0-1.567-.524Zm23.972 2.035a2.5 2.5 0 0 0-1.567.524a2.4 2.4 0 0 0-.805 1.361a16.5 16.5 0 0 0-.212 3.109a24 24 0 0 1-.169 3.234a3.35 3.35 0 0 1-.681 1.63a2.97 2.97 0 0 1-1.324.93a5.7 5.7 0 0 1-1.773.2h-.7V26.04h.387a2.64 2.64 0 0 0 1.592-.337a1.76 1.76 0 0 0 .506-1.186q.05-.462.05-2.9a8.9 8.9 0 0 1 .618-3.865A3.8 3.8 0 0 1 25.9 16a4.6 4.6 0 0 1-1.7-1.286a4.7 4.7 0 0 1-.768-2.06a36 36 0 0 1-.137-4.133a3.4 3.4 0 0 0-.425-2.092a2.34 2.34 0 0 0-1.723-.468h-.387V4h.7a6.8 6.8 0 0 1 1.54.125a3.05 3.05 0 0 1 1.149.581a3 3 0 0 1 .755 1.018a5.2 5.2 0 0 1 .418 1.686q.062.662.075 2.747a15.3 15.3 0 0 0 .212 2.947a2.38 2.38 0 0 0 .805 1.355a2.5 2.5 0 0 0 1.567.518Z\"\u002F>",{"id":31,"title":32,"body":33,"description":674,"extension":675,"meta":676,"navigation":10,"path":677,"seo":678,"stem":679,"__hash__":680},"docs\u002Fdocs\u002Fv2\u002Fbench\u002Ftests.md","tests\u002F",{"type":34,"value":35,"toc":666},"minimark",[36,40,49,254,257,262,267,270,301,309,313,316,380,393,397,403,483,493,496,555,559,562,597,604,608,611,645,648,652,655,662],[37,38,32],"h1",{"id":39},"tests",[41,42,43,44,48],"p",{},"Benchmark scenarios are standard Bun test files. No special harness, no benchmark-specific\nruntime wrapper — you boot ",[45,46,47],"code",{},"Axon()"," exactly as you would in an ordinary agent test.",[50,51,56],"pre",{"className":52,"code":53,"language":54,"meta":55,"style":55},"language-ts shiki shiki-themes dark-plus","it(\"resolves the failing test\", async ({ workspace }) => {\n    expect(workspace).toHaveFile(\"src\u002Fparser.ts\")\n\n    const { axon } = await Axon()\n    await axon.request(\"Fix the failing test in src\u002Fparser.ts\")\n\n    observe(\"resolved\", await workspace.tests.pass())\n    observe(\"collateral\", (await workspace.changed()).length - 1)\n})\n","ts","",[45,57,58,98,122,128,154,176,181,211,248],{"__ignoreMap":55},[59,60,63,67,71,75,78,82,85,89,92,95],"span",{"class":61,"line":62},"line",1,[59,64,66],{"class":65},"sCudf","it",[59,68,70],{"class":69},"sTNBD","(",[59,72,74],{"class":73},"sKc5r","\"resolves the failing test\"",[59,76,77],{"class":69},", ",[59,79,81],{"class":80},"scz_3","async",[59,83,84],{"class":69}," ({ ",[59,86,88],{"class":87},"s3F5K","workspace",[59,90,91],{"class":69}," }) ",[59,93,94],{"class":80},"=>",[59,96,97],{"class":69}," {\n",[59,99,101,104,106,108,111,114,116,119],{"class":61,"line":100},2,[59,102,103],{"class":65},"    expect",[59,105,70],{"class":69},[59,107,88],{"class":87},[59,109,110],{"class":69},").",[59,112,113],{"class":65},"toHaveFile",[59,115,70],{"class":69},[59,117,118],{"class":73},"\"src\u002Fparser.ts\"",[59,120,121],{"class":69},")\n",[59,123,125],{"class":61,"line":124},3,[59,126,127],{"emptyLinePlaceholder":10},"\n",[59,129,131,134,137,141,144,148,151],{"class":61,"line":130},4,[59,132,133],{"class":80},"    const",[59,135,136],{"class":69}," { ",[59,138,140],{"class":139},"s9McN","axon",[59,142,143],{"class":69}," } = ",[59,145,147],{"class":146},"sYbnZ","await",[59,149,150],{"class":65}," Axon",[59,152,153],{"class":69},"()\n",[59,155,157,160,163,166,169,171,174],{"class":61,"line":156},5,[59,158,159],{"class":146},"    await",[59,161,162],{"class":87}," axon",[59,164,165],{"class":69},".",[59,167,168],{"class":65},"request",[59,170,70],{"class":69},[59,172,173],{"class":73},"\"Fix the failing test in src\u002Fparser.ts\"",[59,175,121],{"class":69},[59,177,179],{"class":61,"line":178},6,[59,180,127],{"emptyLinePlaceholder":10},[59,182,184,187,189,192,194,196,199,201,203,205,208],{"class":61,"line":183},7,[59,185,186],{"class":65},"    observe",[59,188,70],{"class":69},[59,190,191],{"class":73},"\"resolved\"",[59,193,77],{"class":69},[59,195,147],{"class":146},[59,197,198],{"class":87}," workspace",[59,200,165],{"class":69},[59,202,39],{"class":87},[59,204,165],{"class":69},[59,206,207],{"class":65},"pass",[59,209,210],{"class":69},"())\n",[59,212,214,216,218,221,224,226,228,230,233,236,239,242,246],{"class":61,"line":213},8,[59,215,186],{"class":65},[59,217,70],{"class":69},[59,219,220],{"class":73},"\"collateral\"",[59,222,223],{"class":69},", (",[59,225,147],{"class":146},[59,227,198],{"class":87},[59,229,165],{"class":69},[59,231,232],{"class":65},"changed",[59,234,235],{"class":69},"()).",[59,237,238],{"class":87},"length",[59,240,241],{"class":69}," - ",[59,243,245],{"class":244},"sDGX8","1",[59,247,121],{"class":69},[59,249,251],{"class":61,"line":250},9,[59,252,253],{"class":69},"})\n",[41,255,256],{},"That file runs once per cell per trial. With two models and three trials, it runs six\ntimes, each against a fresh copy of the workspace and an agent configured for that cell.",[258,259,261],"h2",{"id":260},"the-test-does-not-know-which-cell-it-is-in","The test does not know which cell it is in",[41,263,264,266],{},[45,265,47],{}," boots the subject with the current variation already applied. Nothing in the test\nselects a model or names a cognet, and that is deliberate: a scenario that branched on which\nmodel it received would no longer be the same scenario across cells, and the comparison\nwould mean nothing.",[41,268,269],{},"If you need the current variation for a label or a path, it is available and typed from\nyour own matrix:",[50,271,273],{"className":52,"code":272,"language":54,"meta":55,"style":55},"const engine = bench.axis(\"model\")\n",[45,274,275],{"__ignoreMap":55},[59,276,277,280,283,286,289,291,294,296,299],{"class":61,"line":62},[59,278,279],{"class":80},"const",[59,281,282],{"class":139}," engine",[59,284,285],{"class":69}," = ",[59,287,288],{"class":87},"bench",[59,290,165],{"class":69},[59,292,293],{"class":65},"axis",[59,295,70],{"class":69},[59,297,298],{"class":73},"\"model\"",[59,300,121],{"class":69},[41,302,303,304,308],{},"Reach for it rarely. Wanting to ",[305,306,307],"em",{},"behave"," differently per cell is usually a sign the\nbenchmark should be two benchmarks.",[258,310,312],{"id":311},"expect-guards-observe-records","expect guards, observe records",[41,314,315],{},"The one rule that separates a benchmark from a test suite:",[50,317,319],{"className":52,"code":318,"language":54,"meta":55,"style":55},"\u002F\u002F The scenario must hold, or this trial measured nothing.\nexpect(workspace).toHaveFile(\"src\u002Fparser.ts\")\n\n\u002F\u002F The subject did something. Record it whatever it was.\nobserve(\"resolved\", await workspace.tests.pass())\n",[45,320,321,327,346,350,355],{"__ignoreMap":55},[59,322,323],{"class":61,"line":62},[59,324,326],{"class":325},"sOLPB","\u002F\u002F The scenario must hold, or this trial measured nothing.\n",[59,328,329,332,334,336,338,340,342,344],{"class":61,"line":100},[59,330,331],{"class":65},"expect",[59,333,70],{"class":69},[59,335,88],{"class":87},[59,337,110],{"class":69},[59,339,113],{"class":65},[59,341,70],{"class":69},[59,343,118],{"class":73},[59,345,121],{"class":69},[59,347,348],{"class":61,"line":124},[59,349,127],{"emptyLinePlaceholder":10},[59,351,352],{"class":61,"line":130},[59,353,354],{"class":325},"\u002F\u002F The subject did something. Record it whatever it was.\n",[59,356,357,360,362,364,366,368,370,372,374,376,378],{"class":61,"line":156},[59,358,359],{"class":65},"observe",[59,361,70],{"class":69},[59,363,191],{"class":73},[59,365,77],{"class":69},[59,367,147],{"class":146},[59,369,198],{"class":87},[59,371,165],{"class":69},[59,373,39],{"class":87},[59,375,165],{"class":69},[59,377,207],{"class":65},[59,379,210],{"class":69},[41,381,382,384,385,387,388,165],{},[45,383,331],{}," throwing excludes the trial rather than scoring it zero. ",[45,386,359],{}," never throws,\nso a model that fails every trial produces a complete record of failing — which is the\nfinding, not a broken benchmark. See ",[389,390,392],"a",{"href":391},"\u002Fdocs\u002Fv2\u002Fbench\u002Fmeasuring","Measuring",[258,394,396],{"id":395},"the-workspace-handle","The workspace handle",[41,398,399,400,402],{},"Tests receive ",[45,401,88],{}," as a test context argument — the agent's actual working directory\nfor this trial, plus the things a benchmark usually wants to ask about it:",[50,404,406],{"className":52,"code":405,"language":54,"meta":55,"style":55},"await workspace.tests.pass()      \u002F\u002F run the workspace's own suite\nawait workspace.changed()         \u002F\u002F files the agent touched\nawait workspace.diff()            \u002F\u002F the full patch\nawait workspace.read(\"src\u002Fx.ts\")  \u002F\u002F read a file\n",[45,407,408,428,444,461],{"__ignoreMap":55},[59,409,410,412,414,416,418,420,422,425],{"class":61,"line":62},[59,411,147],{"class":146},[59,413,198],{"class":87},[59,415,165],{"class":69},[59,417,39],{"class":87},[59,419,165],{"class":69},[59,421,207],{"class":65},[59,423,424],{"class":69},"()      ",[59,426,427],{"class":325},"\u002F\u002F run the workspace's own suite\n",[59,429,430,432,434,436,438,441],{"class":61,"line":100},[59,431,147],{"class":146},[59,433,198],{"class":87},[59,435,165],{"class":69},[59,437,232],{"class":65},[59,439,440],{"class":69},"()         ",[59,442,443],{"class":325},"\u002F\u002F files the agent touched\n",[59,445,446,448,450,452,455,458],{"class":61,"line":124},[59,447,147],{"class":146},[59,449,198],{"class":87},[59,451,165],{"class":69},[59,453,454],{"class":65},"diff",[59,456,457],{"class":69},"()            ",[59,459,460],{"class":325},"\u002F\u002F the full patch\n",[59,462,463,465,467,469,472,474,477,480],{"class":61,"line":130},[59,464,147],{"class":146},[59,466,198],{"class":87},[59,468,165],{"class":69},[59,470,471],{"class":65},"read",[59,473,70],{"class":69},[59,475,476],{"class":73},"\"src\u002Fx.ts\"",[59,478,479],{"class":69},")  ",[59,481,482],{"class":325},"\u002F\u002F read a file\n",[41,484,485,488,489,492],{},[45,486,487],{},"changed()"," and ",[45,490,491],{},"diff()"," work because the harness snapshots the world before the agent\nboots. A test could not compute either on its own — by the time it runs, the original is\ngone.",[41,494,495],{},"Matchers are available on the same handle:",[50,497,499],{"className":52,"code":498,"language":54,"meta":55,"style":55},"expect(workspace).toHaveFile(\"src\u002Fparser.ts\")\nexpect(workspace).toPassTests()\nexpect(workspace).toHaveChangedOnly([\"src\u002Fparser.ts\"])\n",[45,500,501,519,534],{"__ignoreMap":55},[59,502,503,505,507,509,511,513,515,517],{"class":61,"line":62},[59,504,331],{"class":65},[59,506,70],{"class":69},[59,508,88],{"class":87},[59,510,110],{"class":69},[59,512,113],{"class":65},[59,514,70],{"class":69},[59,516,118],{"class":73},[59,518,121],{"class":69},[59,520,521,523,525,527,529,532],{"class":61,"line":100},[59,522,331],{"class":65},[59,524,70],{"class":69},[59,526,88],{"class":87},[59,528,110],{"class":69},[59,530,531],{"class":65},"toPassTests",[59,533,153],{"class":69},[59,535,536,538,540,542,544,547,550,552],{"class":61,"line":124},[59,537,331],{"class":65},[59,539,70],{"class":69},[59,541,88],{"class":87},[59,543,110],{"class":69},[59,545,546],{"class":65},"toHaveChangedOnly",[59,548,549],{"class":69},"([",[59,551,118],{"class":73},[59,553,554],{"class":69},"])\n",[258,556,558],{"id":557},"multiple-scenarios","Multiple scenarios",[41,560,561],{},"One file per scenario, named for what it asks:",[50,563,567],{"className":564,"code":565,"language":566,"meta":55,"style":55},"language-bash shiki shiki-themes dark-plus","tests\u002F\n├── add-feature.bench.ts\n├── fix-bug.bench.ts\n└── refactor.bench.ts\n","bash",[45,568,569,574,582,589],{"__ignoreMap":55},[59,570,571],{"class":61,"line":62},[59,572,573],{"class":65},"tests\u002F\n",[59,575,576,579],{"class":61,"line":100},[59,577,578],{"class":65},"├──",[59,580,581],{"class":73}," add-feature.bench.ts\n",[59,583,584,586],{"class":61,"line":124},[59,585,578],{"class":65},[59,587,588],{"class":73}," fix-bug.bench.ts\n",[59,590,591,594],{"class":61,"line":130},[59,592,593],{"class":65},"└──",[59,595,596],{"class":73}," refactor.bench.ts\n",[41,598,599,600,603],{},"Every scenario runs in every cell, and results are recorded per test. That is what lets a\nresult say ",[305,601,602],{},"this model wins overall but loses on refactoring"," — a single combined score\nwould have hidden it.",[258,605,607],{"id":606},"attaching-evidence","Attaching evidence",[41,609,610],{},"Numbers say which cell won. Artifacts say what actually happened:",[50,612,614],{"className":52,"code":613,"language":54,"meta":55,"style":55},"await bench.attach(\"diff\", await workspace.diff())\n",[45,615,616],{"__ignoreMap":55},[59,617,618,620,623,625,628,630,633,635,637,639,641,643],{"class":61,"line":62},[59,619,147],{"class":146},[59,621,622],{"class":87}," bench",[59,624,165],{"class":69},[59,626,627],{"class":65},"attach",[59,629,70],{"class":69},[59,631,632],{"class":73},"\"diff\"",[59,634,77],{"class":69},[59,636,147],{"class":146},[59,638,198],{"class":87},[59,640,165],{"class":69},[59,642,454],{"class":65},[59,644,210],{"class":69},[41,646,647],{},"Stored content-addressed, deduplicated by hash, and linked to the trial that produced it.",[258,649,651],{"id":650},"trials-and-determinism","Trials and determinism",[41,653,654],{},"Each trial is a fresh workspace and a fresh agent. Nothing carries between them — no\nfiles, no session, no memory.",[41,656,657,658,661],{},"Which means ",[45,659,660],{},"trials: 1"," is measuring a single sample of a stochastic system. Models vary\nrun to run; one sample is an anecdote. Three is a minimum for noticing variance at all, and\nten or more is where a difference between two cells starts to mean something.",[663,664,665],"style",{},"html pre.shiki code .sCudf, html code.shiki .sCudf{--shiki-default:#DCDCAA}html pre.shiki code .sTNBD, html code.shiki .sTNBD{--shiki-default:#D4D4D4}html pre.shiki code .sKc5r, html code.shiki .sKc5r{--shiki-default:#CE9178}html pre.shiki code .scz_3, html code.shiki .scz_3{--shiki-default:#569CD6}html pre.shiki code .s3F5K, html code.shiki .s3F5K{--shiki-default:#9CDCFE}html pre.shiki code .s9McN, html code.shiki .s9McN{--shiki-default:#4FC1FF}html pre.shiki code .sYbnZ, html code.shiki .sYbnZ{--shiki-default:#C586C0}html pre.shiki code .sDGX8, html code.shiki .sDGX8{--shiki-default:#B5CEA8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sOLPB, html code.shiki .sOLPB{--shiki-default:#6A9955}",{"title":55,"searchDepth":100,"depth":100,"links":667},[668,669,670,671,672,673],{"id":260,"depth":100,"text":261},{"id":311,"depth":100,"text":312},{"id":395,"depth":100,"text":396},{"id":557,"depth":100,"text":558},{"id":606,"depth":100,"text":607},{"id":650,"depth":100,"text":651},"The scenarios — ordinary Bun tests that boot an agent and record what happened.","md",{},"\u002Fdocs\u002Fv2\u002Fbench\u002Ftests",{"title":32,"description":674},"docs\u002Fv2\u002Fbench\u002Ftests","NBgVpuUqryLl7tYiFQnbhsJC0dl0dR5LnS1Mv6leFnI",1788121883366]