[{"data":1,"prerenderedAt":656},["ShallowReactive",2],{"i-mdi:magnify":3,"i-simple-icons:visualstudiocode":8,"i-mdi:github":11,"i-mdi:discord":13,"i-mdi:youtube":15,"i-mdi:flask-outline":17,"i-vscode-icons:default-folder-opened":19,"i-vscode-icons:default-folder":22,"i-vscode-icons:folder-type-test":24,"i-vscode-icons:file-type-typescript":26,"i-vscode-icons:file-type-json":28,"docs-\u002Fdocs\u002Fv2\u002Fbench\u002Fconfig":30},{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":7},0,24,false,"\u003Cpath fill=\"currentColor\" d=\"M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5l-1.5 1.5l-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16A6.5 6.5 0 0 1 3 9.5A6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14S14 12 14 9.5S12 5 9.5 5\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":9,"hidden":10},"\u003Cpath fill=\"currentColor\" d=\"M23.15 2.587L18.21.21a1.49 1.49 0 0 0-1.705.29l-9.46 8.63l-4.12-3.128a1 1 0 0 0-1.276.057L.327 7.261A1 1 0 0 0 .326 8.74L3.899 12L.326 15.26a1 1 0 0 0 .001 1.479L1.65 17.94a1 1 0 0 0 1.276.057l4.12-3.128l9.46 8.63a1.49 1.49 0 0 0 1.704.29l4.942-2.377A1.5 1.5 0 0 0 24 20.06V3.939a1.5 1.5 0 0 0-.85-1.352m-5.146 14.861L10.826 12l7.178-5.448z\"\u002F>",true,{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":12},"\u003Cpath fill=\"currentColor\" d=\"M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5c.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34c-.46-1.16-1.11-1.47-1.11-1.47c-.91-.62.07-.6.07-.6c1 .07 1.53 1.03 1.53 1.03c.87 1.52 2.34 1.07 2.91.83c.09-.65.35-1.09.63-1.34c-2.22-.25-4.55-1.11-4.55-4.92c0-1.11.38-2 1.03-2.71c-.1-.25-.45-1.29.1-2.64c0 0 .84-.27 2.75 1.02c.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02c.55 1.35.2 2.39.1 2.64c.65.71 1.03 1.6 1.03 2.71c0 3.82-2.34 4.66-4.57 4.91c.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":14,"hidden":10},"\u003Cpath fill=\"currentColor\" d=\"m22 24l-5.25-5l.63 2H4.5A2.5 2.5 0 0 1 2 18.5v-15A2.5 2.5 0 0 1 4.5 1h15A2.5 2.5 0 0 1 22 3.5V24M12 6.8c-2.68 0-4.56 1.15-4.56 1.15c1.03-.92 2.83-1.45 2.83-1.45l-.17-.17c-1.69.03-3.22 1.2-3.22 1.2c-1.72 3.59-1.61 6.69-1.61 6.69c1.4 1.81 3.48 1.68 3.48 1.68l.71-.9c-1.25-.27-2.04-1.38-2.04-1.38S9.3 14.9 12 14.9s4.58-1.28 4.58-1.28s-.79 1.11-2.04 1.38l.71.9s2.08.13 3.48-1.68c0 0 .11-3.1-1.61-6.69c0 0-1.53-1.17-3.22-1.2l-.17.17s1.8.53 2.83 1.45c0 0-1.88-1.15-4.56-1.15m-2.07 3.79c.65 0 1.18.57 1.17 1.27c0 .69-.52 1.27-1.17 1.27c-.64 0-1.16-.58-1.16-1.27c0-.7.51-1.27 1.16-1.27m4.17 0c.65 0 1.17.57 1.17 1.27c0 .69-.52 1.27-1.17 1.27c-.64 0-1.16-.58-1.16-1.27c0-.7.51-1.27 1.16-1.27Z\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":16},"\u003Cpath fill=\"currentColor\" d=\"m10 15l5.19-3L10 9zm11.56-7.83c.13.47.22 1.1.28 1.9c.07.8.1 1.49.1 2.09L22 12c0 2.19-.16 3.8-.44 4.83c-.25.9-.83 1.48-1.73 1.73c-.47.13-1.33.22-2.65.28c-1.3.07-2.49.1-3.59.1L12 19c-4.19 0-6.8-.16-7.83-.44c-.9-.25-1.48-.83-1.73-1.73c-.13-.47-.22-1.1-.28-1.9c-.07-.8-.1-1.49-.1-2.09L2 12c0-2.19.16-3.8.44-4.83c.25-.9.83-1.48 1.73-1.73c.47-.13 1.33-.22 2.65-.28c1.3-.07 2.49-.1 3.59-.1L12 5c4.19 0 6.8.16 7.83.44c.9.25 1.48.83 1.73 1.73\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":18},"\u003Cpath fill=\"currentColor\" d=\"M5 19a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1c0-.21-.07-.41-.18-.57L13 8.35V4h-2v4.35L5.18 18.43c-.11.16-.18.36-.18.57m1 3a3 3 0 0 1-3-3c0-.6.18-1.16.5-1.63L9 7.81V6a1 1 0 0 1-1-1V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v1a1 1 0 0 1-1 1v1.81l5.5 9.56c.32.47.5 1.03.5 1.63a3 3 0 0 1-3 3zm7-6l1.34-1.34L16.27 18H7.73l2.66-4.61zm-.5-4a.5.5 0 0 1 .5.5a.5.5 0 0 1-.5.5a.5.5 0 0 1-.5-.5a.5.5 0 0 1 .5-.5\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":21},32,"\u003Cpath fill=\"#dcb67a\" d=\"M27.4 5.5h-9.2l-2.1 4.2H4.3v16.8h25.2v-21Zm0 18.7H6.6V11.8h20.8Zm0-14.5h-8.2l1-2.1h7.1v2.1Z\"\u002F>\u003Cpath fill=\"#dcb67a\" d=\"M25.7 13.7H.5l3.8 12.8h25.2z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":23},"\u003Cpath fill=\"#c09553\" d=\"M27.5 5.5h-9.3l-2.1 4.2H4.4v16.8h25.2v-21Zm0 4.2h-8.2l1.1-2.1h7.1Z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":25},"\u003Cpath fill=\"#b70b0b\" d=\"M27.5 5.5h-9.3l-2.1 4.2H4.4v16.8h25.2v-21Zm0 4.2h-8.2l1.1-2.1h7.1Z\"\u002F>\u003Cpath fill=\"#fac6c5\" d=\"M28.3 27.423c-1.729-.4-2.332-2.482-1.329-4.583a8.95 8.95 0 0 1 2.217-2.779l.314-.26l.009.357a6.7 6.7 0 0 0 .748 2.425a14 14 0 0 1 .625 1.631a3.2 3.2 0 0 1-.073 1.958a2.67 2.67 0 0 1-1.111 1.087a2.8 2.8 0 0 1-1.4.164m.351-3.565c.124-.143.155-.332.164-1l.011-.825l-.2.158a1.68 1.68 0 0 0-.635 1.4c.07.214.286.448.411.446a.5.5 0 0 0 .249-.179m-15.527 7.087a3.85 3.85 0 0 1-3.031-4.563c.254-1.07.2-1.008 5.726-6.787l5.2-5.442l.738.7l.738.7l-3 3.132l-3 3.132l2.508-.054l2.508-.054l1.773-1.85l1.773-1.85l.735.707l.735.707l-5.213 5.43c-4.783 4.982-5.257 5.452-5.743 5.692a3.9 3.9 0 0 1-2.452.393Zm14.662-12.484c-.138-.082-1.546-1.4-3.129-2.921l-2.879-2.772l-.009-.421a.86.86 0 0 1 .194-.673A.99.99 0 0 1 23 11.3a69 69 0 0 1 6.277 6.023a1 1 0 0 1-1.492 1.133Z\"\u002F>\u003Cpath fill=\"#fac6c5\" d=\"M11.578 26.851a2.1 2.1 0 0 0 2.579 2.614c.191-.2 2.407-2.515 7.387-7.737c-2.505.054-2.475-.055-4.98 0c-5.015 5.195-5.114 5.157-4.986 5.123\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":27},"\u003Cpath fill=\"#007acc\" d=\"M23.827 8.243a4.4 4.4 0 0 1 2.223 1.281a6 6 0 0 1 .852 1.143c.011.045-1.534 1.083-2.471 1.662c-.034.023-.169-.124-.322-.35a2.01 2.01 0 0 0-1.67-1c-1.077-.074-1.771.49-1.766 1.433a1.3 1.3 0 0 0 .153.666c.237.49.677.784 2.059 1.383c2.544 1.095 3.636 1.817 4.31 2.843a5.16 5.16 0 0 1 .416 4.333a4.76 4.76 0 0 1-3.932 2.815a11 11 0 0 1-2.708-.028a6.53 6.53 0 0 1-3.616-1.884a6.3 6.3 0 0 1-.926-1.371a3 3 0 0 1 .327-.208c.158-.09.756-.434 1.32-.761l1.024-.6l.214.312a4.8 4.8 0 0 0 1.35 1.292a3.3 3.3 0 0 0 3.458-.175a1.545 1.545 0 0 0 .2-1.974c-.276-.395-.84-.727-2.443-1.422a8.8 8.8 0 0 1-3.349-2.055a4.7 4.7 0 0 1-.976-1.777a7.1 7.1 0 0 1-.062-2.268a4.33 4.33 0 0 1 3.644-3.374a9 9 0 0 1 2.691.084m-8.343 1.483l.011 1.454h-4.63v13.148H7.6V11.183H2.97V9.755a14 14 0 0 1 .04-1.466c.017-.023 2.832-.034 6.245-.028l6.211.017Z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":29},"\u003Cpath fill=\"#f5de19\" d=\"M4.014 14.976a2.5 2.5 0 0 0 1.567-.518a2.38 2.38 0 0 0 .805-1.358a15.3 15.3 0 0 0 .214-2.944q.012-2.085.075-2.747a5.2 5.2 0 0 1 .418-1.686a3 3 0 0 1 .755-1.018A3.05 3.05 0 0 1 9 4.125A6.8 6.8 0 0 1 10.544 4h.7v1.96h-.387a2.34 2.34 0 0 0-1.723.468a3.4 3.4 0 0 0-.425 2.092a36 36 0 0 1-.137 4.133a4.7 4.7 0 0 1-.768 2.06A4.6 4.6 0 0 1 6.1 16a3.8 3.8 0 0 1 1.992 1.754a8.9 8.9 0 0 1 .618 3.865q0 2.435.05 2.9a1.76 1.76 0 0 0 .504 1.181a2.64 2.64 0 0 0 1.592.337h.387V28h-.7a5.7 5.7 0 0 1-1.773-.2a2.97 2.97 0 0 1-1.324-.93a3.35 3.35 0 0 1-.681-1.63a24 24 0 0 1-.165-3.234a16.5 16.5 0 0 0-.214-3.106a2.4 2.4 0 0 0-.805-1.361a2.5 2.5 0 0 0-1.567-.524Zm23.972 2.035a2.5 2.5 0 0 0-1.567.524a2.4 2.4 0 0 0-.805 1.361a16.5 16.5 0 0 0-.212 3.109a24 24 0 0 1-.169 3.234a3.35 3.35 0 0 1-.681 1.63a2.97 2.97 0 0 1-1.324.93a5.7 5.7 0 0 1-1.773.2h-.7V26.04h.387a2.64 2.64 0 0 0 1.592-.337a1.76 1.76 0 0 0 .506-1.186q.05-.462.05-2.9a8.9 8.9 0 0 1 .618-3.865A3.8 3.8 0 0 1 25.9 16a4.6 4.6 0 0 1-1.7-1.286a4.7 4.7 0 0 1-.768-2.06a36 36 0 0 1-.137-4.133a3.4 3.4 0 0 0-.425-2.092a2.34 2.34 0 0 0-1.723-.468h-.387V4h.7a6.8 6.8 0 0 1 1.54.125a3.05 3.05 0 0 1 1.149.581a3 3 0 0 1 .755 1.018a5.2 5.2 0 0 1 .418 1.686q.062.662.075 2.747a15.3 15.3 0 0 0 .212 2.947a2.38 2.38 0 0 0 .805 1.355a2.5 2.5 0 0 0 1.567.518Z\"\u002F>",{"id":31,"title":32,"body":33,"description":649,"extension":650,"meta":651,"navigation":10,"path":652,"seo":653,"stem":654,"__hash__":655},"docs\u002Fdocs\u002Fv2\u002Fbench\u002Fconfig.md","bench.config.ts",{"type":34,"value":35,"toc":641},"minimark",[36,40,62,315,318,328,333,418,425,427,442,445,448,450,453,507,510,513,515,548,557,560,564,609,615,619,622,637],[37,38,32],"h1",{"id":39},"benchconfigts",[41,42,43,44,48,49,52,53,61],"p",{},"The config declares what ",[45,46,47],"em",{},"varies"," and what is ",[45,50,51],{},"measured",". Nothing about what the agent has\nto do — that lives in ",[54,55,57],"a",{"href":56},"\u002Fdocs\u002Fv2\u002Fbench\u002Ftests",[58,59,60],"code",{},"tests\u002F",".",[63,64,69],"pre",{"className":65,"code":66,"language":67,"meta":68,"style":68},"language-ts shiki shiki-themes dark-plus","import { OpenRouter } from \"@arcforge\u002Fengines\"\n\ntype Schema = {\n    \u002F** Has the agent resolved the bug? *\u002F\n    resolved: boolean\n\n    \u002F** Files edited beyond the target. @objective minimize *\u002F\n    collateral: number\n}\n\nexport default defineBench\u003CSchema>({\n    workspace: { source: \".\u002Fworkspace\", retain: \"failed\" },\n\n    matrix: {\n        model: [\n            OpenRouter({ model: \"anthropic\u002Fclaude-sonnet-4.6\" }),\n            OpenRouter({ model: \"openai\u002Fgpt-5\" }),\n        ],\n    },\n\n    trials: 3,\n})\n","ts","",[58,70,71,98,104,118,125,137,142,154,165,171,176,198,224,229,238,247,265,279,285,291,296,309],{"__ignoreMap":68},[72,73,76,80,84,88,91,94],"span",{"class":74,"line":75},"line",1,[72,77,79],{"class":78},"sYbnZ","import",[72,81,83],{"class":82},"sTNBD"," { ",[72,85,87],{"class":86},"s3F5K","OpenRouter",[72,89,90],{"class":82}," } ",[72,92,93],{"class":78},"from",[72,95,97],{"class":96},"sKc5r"," \"@arcforge\u002Fengines\"\n",[72,99,101],{"class":74,"line":100},2,[72,102,103],{"emptyLinePlaceholder":10},"\n",[72,105,107,111,115],{"class":74,"line":106},3,[72,108,110],{"class":109},"scz_3","type",[72,112,114],{"class":113},"sNl3T"," Schema",[72,116,117],{"class":82}," = {\n",[72,119,121],{"class":74,"line":120},4,[72,122,124],{"class":123},"sOLPB","    \u002F** Has the agent resolved the bug? *\u002F\n",[72,126,128,131,134],{"class":74,"line":127},5,[72,129,130],{"class":86},"    resolved",[72,132,133],{"class":82},": ",[72,135,136],{"class":113},"boolean\n",[72,138,140],{"class":74,"line":139},6,[72,141,103],{"emptyLinePlaceholder":10},[72,143,145,148,151],{"class":74,"line":144},7,[72,146,147],{"class":123},"    \u002F** Files edited beyond the target. ",[72,149,150],{"class":109},"@objective",[72,152,153],{"class":123}," minimize *\u002F\n",[72,155,157,160,162],{"class":74,"line":156},8,[72,158,159],{"class":86},"    collateral",[72,161,133],{"class":82},[72,163,164],{"class":113},"number\n",[72,166,168],{"class":74,"line":167},9,[72,169,170],{"class":82},"}\n",[72,172,174],{"class":74,"line":173},10,[72,175,103],{"emptyLinePlaceholder":10},[72,177,179,182,185,189,192,195],{"class":74,"line":178},11,[72,180,181],{"class":78},"export",[72,183,184],{"class":78}," default",[72,186,188],{"class":187},"sCudf"," defineBench",[72,190,191],{"class":82},"\u003C",[72,193,194],{"class":113},"Schema",[72,196,197],{"class":82},">({\n",[72,199,201,204,206,209,212,215,218,221],{"class":74,"line":200},12,[72,202,203],{"class":86},"    workspace:",[72,205,83],{"class":82},[72,207,208],{"class":86},"source:",[72,210,211],{"class":96}," \".\u002Fworkspace\"",[72,213,214],{"class":82},", ",[72,216,217],{"class":86},"retain:",[72,219,220],{"class":96}," \"failed\"",[72,222,223],{"class":82}," },\n",[72,225,227],{"class":74,"line":226},13,[72,228,103],{"emptyLinePlaceholder":10},[72,230,232,235],{"class":74,"line":231},14,[72,233,234],{"class":86},"    matrix:",[72,236,237],{"class":82}," {\n",[72,239,241,244],{"class":74,"line":240},15,[72,242,243],{"class":86},"        model:",[72,245,246],{"class":82}," [\n",[72,248,250,253,256,259,262],{"class":74,"line":249},16,[72,251,252],{"class":187},"            OpenRouter",[72,254,255],{"class":82},"({ ",[72,257,258],{"class":86},"model:",[72,260,261],{"class":96}," \"anthropic\u002Fclaude-sonnet-4.6\"",[72,263,264],{"class":82}," }),\n",[72,266,268,270,272,274,277],{"class":74,"line":267},17,[72,269,252],{"class":187},[72,271,255],{"class":82},[72,273,258],{"class":86},[72,275,276],{"class":96}," \"openai\u002Fgpt-5\"",[72,278,264],{"class":82},[72,280,282],{"class":74,"line":281},18,[72,283,284],{"class":82},"        ],\n",[72,286,288],{"class":74,"line":287},19,[72,289,290],{"class":82},"    },\n",[72,292,294],{"class":74,"line":293},20,[72,295,103],{"emptyLinePlaceholder":10},[72,297,299,302,306],{"class":74,"line":298},21,[72,300,301],{"class":86},"    trials:",[72,303,305],{"class":304},"sDGX8"," 3",[72,307,308],{"class":82},",\n",[72,310,312],{"class":74,"line":311},22,[72,313,314],{"class":82},"})\n",[41,316,317],{},"Nothing is required. A benchmark with no matrix runs one cell; with no workspace it runs\nagainst an empty world.",[41,319,320,321,327],{},"Identity is not here. Name, version and description live in\n",[54,322,324],{"href":323},"\u002Fdocs\u002Fv2\u002Fbench\u002Fpackage",[58,325,326],{},"package.json"," — stating them twice is how the two come to\ndisagree.",[329,330,332],"h2",{"id":331},"fields","Fields",[334,335,336,347],"table",{},[337,338,339],"thead",{},[340,341,342,345],"tr",{},[343,344],"th",{},[343,346],{},[348,349,350,365,375,385,398,408],"tbody",{},[340,351,352,358],{},[353,354,355],"td",{},[58,356,357],{},"matrix",[353,359,360,361],{},"what varies — see ",[54,362,364],{"href":363},"\u002Fdocs\u002Fv2\u002Fbench\u002Fmatrix","The Matrix",[340,366,367,372],{},[353,368,369],{},[58,370,371],{},"trials",[353,373,374],{},"repetitions per test × cell. Defaults to 1",[340,376,377,382],{},[353,378,379],{},[58,380,381],{},"workspace",[353,383,384],{},"the agent's world and its retention policy",[340,386,387,392],{},[353,388,389],{},[58,390,391],{},"tests",[353,393,394,395],{},"globs for scenario files. Defaults to ",[58,396,397],{},"tests\u002F**\u002F*.bench.ts",[340,399,400,405],{},[353,401,402],{},[58,403,404],{},"setup",[353,406,407],{},"runs after the workspace copy, before the agent boots",[340,409,410,415],{},[353,411,412],{},[58,413,414],{},"budget",[353,416,417],{},"spend and time limits, per trial or per run",[41,419,420,421,61],{},"The measurement schema is the type parameter, not a field. That is\n",[54,422,424],{"href":423},"\u002Fdocs\u002Fv2\u002Fbench\u002Fmeasuring","Measuring",[329,426,371],{"id":371},[63,428,430],{"className":65,"code":429,"language":67,"meta":68,"style":68},"trials: 3\n",[58,431,432],{"__ignoreMap":68},[72,433,434,437,439],{"class":74,"line":75},[72,435,371],{"class":436},"sqDPy",[72,438,133],{"class":82},[72,440,441],{"class":304},"3\n",[41,443,444],{},"Repetitions of every test in every cell. Two models × three trials × one test is six runs.",[41,446,447],{},"The default is 1, and 1 is almost always wrong for a published result. Models are\nstochastic: a single sample tells you what happened once, not what happens. Three is enough\nto notice variance; ten or more is where the difference between two cells starts to carry\nweight.",[329,449,404],{"id":404},[41,451,452],{},"Runs once per iteration, after the workspace has been copied and before the subject boots —\nthe moment when the world exists but nothing has touched it:",[63,454,456],{"className":65,"code":455,"language":67,"meta":68,"style":68},"async setup() {\n    await Bun.$`bun install`.cwd(bench.workspace)\n}\n",[58,457,458,469,503],{"__ignoreMap":68},[72,459,460,463,466],{"class":74,"line":75},[72,461,462],{"class":86},"async",[72,464,465],{"class":187}," setup",[72,467,468],{"class":82},"() {\n",[72,470,471,474,477,479,482,485,487,490,493,496,498,500],{"class":74,"line":100},[72,472,473],{"class":78},"    await",[72,475,476],{"class":86}," Bun",[72,478,61],{"class":82},[72,480,481],{"class":187},"$",[72,483,484],{"class":96},"`bun install`",[72,486,61],{"class":82},[72,488,489],{"class":187},"cwd",[72,491,492],{"class":82},"(",[72,494,495],{"class":86},"bench",[72,497,61],{"class":82},[72,499,381],{"class":86},[72,501,502],{"class":82},")\n",[72,504,505],{"class":74,"line":106},[72,506,170],{"class":82},[41,508,509],{},"Installing dependencies, seeding a database, starting a service the task needs. Anything\nthat must be true of the world before the agent arrives.",[41,511,512],{},"It runs inside every iteration rather than once for the whole run, because a benchmark\nwhose setup leaked state between trials would stop being a controlled experiment.",[329,514,414],{"id":414},[63,516,518],{"className":65,"code":517,"language":67,"meta":68,"style":68},"budget: { perTrial: \"$0.50\", total: \"$25\" }\n",[58,519,520],{"__ignoreMap":68},[72,521,522,524,527,530,532,535,537,540,542,545],{"class":74,"line":75},[72,523,414],{"class":436},[72,525,526],{"class":82},": { ",[72,528,529],{"class":436},"perTrial",[72,531,133],{"class":82},[72,533,534],{"class":96},"\"$0.50\"",[72,536,214],{"class":82},[72,538,539],{"class":436},"total",[72,541,133],{"class":82},[72,543,544],{"class":96},"\"$25\"",[72,546,547],{"class":82}," }\n",[41,549,550,551,553,554,556],{},"A ceiling, not a target. A trial that exceeds ",[58,552,529],{}," is stopped and recorded as a\n",[58,555,414],{}," fault — excluded from scoring rather than counted as a failure, because running out\nof money is a statement about the experiment rather than about the agent.",[41,558,559],{},"Worth setting on anything with a real matrix. Two models × three trials × ten scenarios is\nsixty agent runs, and an agent that loops is a bill that grows while you are not watching.",[329,561,563],{"id":562},"engines-are-imported-not-global","Engines are imported, not global",[63,565,567],{"className":65,"code":566,"language":67,"meta":68,"style":68},"import { OpenRouter, Codex } from \"@arcforge\u002Fengines\"\nimport { Mock, run } from \"@arcforge\u002Fengines\u002Fmock\"\n",[58,568,569,588],{"__ignoreMap":68},[72,570,571,573,575,577,579,582,584,586],{"class":74,"line":75},[72,572,79],{"class":78},[72,574,83],{"class":82},[72,576,87],{"class":86},[72,578,214],{"class":82},[72,580,581],{"class":86},"Codex",[72,583,90],{"class":82},[72,585,93],{"class":78},[72,587,97],{"class":96},[72,589,590,592,594,597,599,602,604,606],{"class":74,"line":100},[72,591,79],{"class":78},[72,593,83],{"class":82},[72,595,596],{"class":86},"Mock",[72,598,214],{"class":82},[72,600,601],{"class":86},"run",[72,603,90],{"class":82},[72,605,93],{"class":78},[72,607,608],{"class":96}," \"@arcforge\u002Fengines\u002Fmock\"\n",[41,610,611,612,614],{},"Engine constructors are ordinary packages. They are declared in the benchmark's\n",[58,613,326],{}," like any dependency, so a published benchmark carries exactly what it needs\nto resolve them.",[329,616,618],{"id":617},"what-does-not-belong-here","What does not belong here",[41,620,621],{},"Scenario logic. If the config is deciding what the agent should do, that belongs in a test.",[41,623,624,625,628,629,632,633,636],{},"Presentation. There is no ",[58,626,627],{},"chart"," or ",[58,630,631],{},"format"," field, and this is deliberate: how a\nmeasurement is drawn follows from what it ",[45,634,635],{},"is"," — a boolean across cells is a bar of rates, a\nnumber with trials is a distribution, two axes make a grid. Declaring the chart as well\nwould mean stating the same fact twice and eventually disagreeing with yourself.",[638,639,640],"style",{},"html pre.shiki code .sYbnZ, html code.shiki .sYbnZ{--shiki-default:#C586C0}html pre.shiki code .sTNBD, html code.shiki .sTNBD{--shiki-default:#D4D4D4}html pre.shiki code .s3F5K, html code.shiki .s3F5K{--shiki-default:#9CDCFE}html pre.shiki code .sKc5r, html code.shiki .sKc5r{--shiki-default:#CE9178}html pre.shiki code .scz_3, html code.shiki .scz_3{--shiki-default:#569CD6}html pre.shiki code .sNl3T, html code.shiki .sNl3T{--shiki-default:#4EC9B0}html pre.shiki code .sOLPB, html code.shiki .sOLPB{--shiki-default:#6A9955}html pre.shiki code .sCudf, html code.shiki .sCudf{--shiki-default:#DCDCAA}html pre.shiki code .sDGX8, html code.shiki .sDGX8{--shiki-default:#B5CEA8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sqDPy, html code.shiki .sqDPy{--shiki-default:#C8C8C8}",{"title":68,"searchDepth":100,"depth":100,"links":642},[643,644,645,646,647,648],{"id":331,"depth":100,"text":332},{"id":371,"depth":100,"text":371},{"id":404,"depth":100,"text":404},{"id":414,"depth":100,"text":414},{"id":562,"depth":100,"text":563},{"id":617,"depth":100,"text":618},"What varies, how many times, and what is measured.","md",{},"\u002Fdocs\u002Fv2\u002Fbench\u002Fconfig",{"title":32,"description":649},"docs\u002Fv2\u002Fbench\u002Fconfig","3I3brMa8S34ojLehUm3CZ0JbPGhAnTxuv-XIzQGD8DU",1788121883683]