[{"data":1,"prerenderedAt":386},["ShallowReactive",2],{"i-mdi:magnify":3,"i-simple-icons:visualstudiocode":8,"i-mdi:github":11,"i-mdi:discord":13,"i-mdi:youtube":15,"i-mdi:flask-outline":17,"i-vscode-icons:default-folder-opened":19,"i-vscode-icons:default-folder":22,"i-vscode-icons:folder-type-test":24,"i-vscode-icons:file-type-typescript":26,"i-vscode-icons:file-type-json":28,"docs-\u002Fdocs\u002Fv2\u002Fbench\u002Ffolder":30},{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":7},0,24,false,"\u003Cpath fill=\"currentColor\" d=\"M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5l-1.5 1.5l-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16A6.5 6.5 0 0 1 3 9.5A6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14S14 12 14 9.5S12 5 9.5 5\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":9,"hidden":10},"\u003Cpath fill=\"currentColor\" d=\"M23.15 2.587L18.21.21a1.49 1.49 0 0 0-1.705.29l-9.46 8.63l-4.12-3.128a1 1 0 0 0-1.276.057L.327 7.261A1 1 0 0 0 .326 8.74L3.899 12L.326 15.26a1 1 0 0 0 .001 1.479L1.65 17.94a1 1 0 0 0 1.276.057l4.12-3.128l9.46 8.63a1.49 1.49 0 0 0 1.704.29l4.942-2.377A1.5 1.5 0 0 0 24 20.06V3.939a1.5 1.5 0 0 0-.85-1.352m-5.146 14.861L10.826 12l7.178-5.448z\"\u002F>",true,{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":12},"\u003Cpath fill=\"currentColor\" d=\"M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5c.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34c-.46-1.16-1.11-1.47-1.11-1.47c-.91-.62.07-.6.07-.6c1 .07 1.53 1.03 1.53 1.03c.87 1.52 2.34 1.07 2.91.83c.09-.65.35-1.09.63-1.34c-2.22-.25-4.55-1.11-4.55-4.92c0-1.11.38-2 1.03-2.71c-.1-.25-.45-1.29.1-2.64c0 0 .84-.27 2.75 1.02c.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02c.55 1.35.2 2.39.1 2.64c.65.71 1.03 1.6 1.03 2.71c0 3.82-2.34 4.66-4.57 4.91c.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":14,"hidden":10},"\u003Cpath fill=\"currentColor\" d=\"m22 24l-5.25-5l.63 2H4.5A2.5 2.5 0 0 1 2 18.5v-15A2.5 2.5 0 0 1 4.5 1h15A2.5 2.5 0 0 1 22 3.5V24M12 6.8c-2.68 0-4.56 1.15-4.56 1.15c1.03-.92 2.83-1.45 2.83-1.45l-.17-.17c-1.69.03-3.22 1.2-3.22 1.2c-1.72 3.59-1.61 6.69-1.61 6.69c1.4 1.81 3.48 1.68 3.48 1.68l.71-.9c-1.25-.27-2.04-1.38-2.04-1.38S9.3 14.9 12 14.9s4.58-1.28 4.58-1.28s-.79 1.11-2.04 1.38l.71.9s2.08.13 3.48-1.68c0 0 .11-3.1-1.61-6.69c0 0-1.53-1.17-3.22-1.2l-.17.17s1.8.53 2.83 1.45c0 0-1.88-1.15-4.56-1.15m-2.07 3.79c.65 0 1.18.57 1.17 1.27c0 .69-.52 1.27-1.17 1.27c-.64 0-1.16-.58-1.16-1.27c0-.7.51-1.27 1.16-1.27m4.17 0c.65 0 1.17.57 1.17 1.27c0 .69-.52 1.27-1.17 1.27c-.64 0-1.16-.58-1.16-1.27c0-.7.51-1.27 1.16-1.27Z\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":16},"\u003Cpath fill=\"currentColor\" d=\"m10 15l5.19-3L10 9zm11.56-7.83c.13.47.22 1.1.28 1.9c.07.8.1 1.49.1 2.09L22 12c0 2.19-.16 3.8-.44 4.83c-.25.9-.83 1.48-1.73 1.73c-.47.13-1.33.22-2.65.28c-1.3.07-2.49.1-3.59.1L12 19c-4.19 0-6.8-.16-7.83-.44c-.9-.25-1.48-.83-1.73-1.73c-.13-.47-.22-1.1-.28-1.9c-.07-.8-.1-1.49-.1-2.09L2 12c0-2.19.16-3.8.44-4.83c.25-.9.83-1.48 1.73-1.73c.47-.13 1.33-.22 2.65-.28c1.3-.07 2.49-.1 3.59-.1L12 5c4.19 0 6.8.16 7.83.44c.9.25 1.48.83 1.73 1.73\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":18},"\u003Cpath fill=\"currentColor\" d=\"M5 19a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1c0-.21-.07-.41-.18-.57L13 8.35V4h-2v4.35L5.18 18.43c-.11.16-.18.36-.18.57m1 3a3 3 0 0 1-3-3c0-.6.18-1.16.5-1.63L9 7.81V6a1 1 0 0 1-1-1V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v1a1 1 0 0 1-1 1v1.81l5.5 9.56c.32.47.5 1.03.5 1.63a3 3 0 0 1-3 3zm7-6l1.34-1.34L16.27 18H7.73l2.66-4.61zm-.5-4a.5.5 0 0 1 .5.5a.5.5 0 0 1-.5.5a.5.5 0 0 1-.5-.5a.5.5 0 0 1 .5-.5\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":21},32,"\u003Cpath fill=\"#dcb67a\" d=\"M27.4 5.5h-9.2l-2.1 4.2H4.3v16.8h25.2v-21Zm0 18.7H6.6V11.8h20.8Zm0-14.5h-8.2l1-2.1h7.1v2.1Z\"\u002F>\u003Cpath fill=\"#dcb67a\" d=\"M25.7 13.7H.5l3.8 12.8h25.2z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":23},"\u003Cpath fill=\"#c09553\" d=\"M27.5 5.5h-9.3l-2.1 4.2H4.4v16.8h25.2v-21Zm0 4.2h-8.2l1.1-2.1h7.1Z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":25},"\u003Cpath fill=\"#b70b0b\" d=\"M27.5 5.5h-9.3l-2.1 4.2H4.4v16.8h25.2v-21Zm0 4.2h-8.2l1.1-2.1h7.1Z\"\u002F>\u003Cpath fill=\"#fac6c5\" d=\"M28.3 27.423c-1.729-.4-2.332-2.482-1.329-4.583a8.95 8.95 0 0 1 2.217-2.779l.314-.26l.009.357a6.7 6.7 0 0 0 .748 2.425a14 14 0 0 1 .625 1.631a3.2 3.2 0 0 1-.073 1.958a2.67 2.67 0 0 1-1.111 1.087a2.8 2.8 0 0 1-1.4.164m.351-3.565c.124-.143.155-.332.164-1l.011-.825l-.2.158a1.68 1.68 0 0 0-.635 1.4c.07.214.286.448.411.446a.5.5 0 0 0 .249-.179m-15.527 7.087a3.85 3.85 0 0 1-3.031-4.563c.254-1.07.2-1.008 5.726-6.787l5.2-5.442l.738.7l.738.7l-3 3.132l-3 3.132l2.508-.054l2.508-.054l1.773-1.85l1.773-1.85l.735.707l.735.707l-5.213 5.43c-4.783 4.982-5.257 5.452-5.743 5.692a3.9 3.9 0 0 1-2.452.393Zm14.662-12.484c-.138-.082-1.546-1.4-3.129-2.921l-2.879-2.772l-.009-.421a.86.86 0 0 1 .194-.673A.99.99 0 0 1 23 11.3a69 69 0 0 1 6.277 6.023a1 1 0 0 1-1.492 1.133Z\"\u002F>\u003Cpath fill=\"#fac6c5\" d=\"M11.578 26.851a2.1 2.1 0 0 0 2.579 2.614c.191-.2 2.407-2.515 7.387-7.737c-2.505.054-2.475-.055-4.98 0c-5.015 5.195-5.114 5.157-4.986 5.123\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":27},"\u003Cpath fill=\"#007acc\" d=\"M23.827 8.243a4.4 4.4 0 0 1 2.223 1.281a6 6 0 0 1 .852 1.143c.011.045-1.534 1.083-2.471 1.662c-.034.023-.169-.124-.322-.35a2.01 2.01 0 0 0-1.67-1c-1.077-.074-1.771.49-1.766 1.433a1.3 1.3 0 0 0 .153.666c.237.49.677.784 2.059 1.383c2.544 1.095 3.636 1.817 4.31 2.843a5.16 5.16 0 0 1 .416 4.333a4.76 4.76 0 0 1-3.932 2.815a11 11 0 0 1-2.708-.028a6.53 6.53 0 0 1-3.616-1.884a6.3 6.3 0 0 1-.926-1.371a3 3 0 0 1 .327-.208c.158-.09.756-.434 1.32-.761l1.024-.6l.214.312a4.8 4.8 0 0 0 1.35 1.292a3.3 3.3 0 0 0 3.458-.175a1.545 1.545 0 0 0 .2-1.974c-.276-.395-.84-.727-2.443-1.422a8.8 8.8 0 0 1-3.349-2.055a4.7 4.7 0 0 1-.976-1.777a7.1 7.1 0 0 1-.062-2.268a4.33 4.33 0 0 1 3.644-3.374a9 9 0 0 1 2.691.084m-8.343 1.483l.011 1.454h-4.63v13.148H7.6V11.183H2.97V9.755a14 14 0 0 1 .04-1.466c.017-.023 2.832-.034 6.245-.028l6.211.017Z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":29},"\u003Cpath fill=\"#f5de19\" d=\"M4.014 14.976a2.5 2.5 0 0 0 1.567-.518a2.38 2.38 0 0 0 .805-1.358a15.3 15.3 0 0 0 .214-2.944q.012-2.085.075-2.747a5.2 5.2 0 0 1 .418-1.686a3 3 0 0 1 .755-1.018A3.05 3.05 0 0 1 9 4.125A6.8 6.8 0 0 1 10.544 4h.7v1.96h-.387a2.34 2.34 0 0 0-1.723.468a3.4 3.4 0 0 0-.425 2.092a36 36 0 0 1-.137 4.133a4.7 4.7 0 0 1-.768 2.06A4.6 4.6 0 0 1 6.1 16a3.8 3.8 0 0 1 1.992 1.754a8.9 8.9 0 0 1 .618 3.865q0 2.435.05 2.9a1.76 1.76 0 0 0 .504 1.181a2.64 2.64 0 0 0 1.592.337h.387V28h-.7a5.7 5.7 0 0 1-1.773-.2a2.97 2.97 0 0 1-1.324-.93a3.35 3.35 0 0 1-.681-1.63a24 24 0 0 1-.165-3.234a16.5 16.5 0 0 0-.214-3.106a2.4 2.4 0 0 0-.805-1.361a2.5 2.5 0 0 0-1.567-.524Zm23.972 2.035a2.5 2.5 0 0 0-1.567.524a2.4 2.4 0 0 0-.805 1.361a16.5 16.5 0 0 0-.212 3.109a24 24 0 0 1-.169 3.234a3.35 3.35 0 0 1-.681 1.63a2.97 2.97 0 0 1-1.324.93a5.7 5.7 0 0 1-1.773.2h-.7V26.04h.387a2.64 2.64 0 0 0 1.592-.337a1.76 1.76 0 0 0 .506-1.186q.05-.462.05-2.9a8.9 8.9 0 0 1 .618-3.865A3.8 3.8 0 0 1 25.9 16a4.6 4.6 0 0 1-1.7-1.286a4.7 4.7 0 0 1-.768-2.06a36 36 0 0 1-.137-4.133a3.4 3.4 0 0 0-.425-2.092a2.34 2.34 0 0 0-1.723-.468h-.387V4h.7a6.8 6.8 0 0 1 1.54.125a3.05 3.05 0 0 1 1.149.581a3 3 0 0 1 .755 1.018a5.2 5.2 0 0 1 .418 1.686q.062.662.075 2.747a15.3 15.3 0 0 0 .212 2.947a2.38 2.38 0 0 0 .805 1.355a2.5 2.5 0 0 0 1.567.518Z\"\u002F>",{"id":31,"title":32,"body":33,"description":379,"extension":380,"meta":381,"navigation":10,"path":382,"seo":383,"stem":384,"__hash__":385},"docs\u002Fdocs\u002Fv2\u002Fbench\u002Ffolder.md","Bench Folder",{"type":34,"value":35,"toc":372},"minimark",[36,40,44,156,167,172,185,191,194,198,218,233,236,240,251,254,304,308,318,321,336,340,359,368],[37,38,32],"h1",{"id":39},"bench-folder",[41,42,43],"p",{},"A benchmark is a folder. Four directories, one config, and every one of them means exactly\none thing.",[45,46,51],"pre",{"className":47,"code":48,"language":49,"meta":50,"style":50},"language-bash shiki shiki-themes dark-plus","my-bench\u002F\n├── .bench\u002F             # generated types, run logs, materialized workspaces\n├── fixtures\u002F           # author-side inputs — never visible to the agent\n├── tests\u002F              # the scenarios\n│   └── fix-bug.bench.ts\n├── workspace\u002F          # the agent's world — copied fresh for every run\n├── bench.config.ts     # what varies, and what is measured\n├── package.json        # name, version, npm deps\n└── README.md           # methodology and limitations\n","bash","",[52,53,54,63,77,88,99,111,122,133,144],"code",{"__ignoreMap":50},[55,56,59],"span",{"class":57,"line":58},"line",1,[55,60,62],{"class":61},"sCudf","my-bench\u002F\n",[55,64,66,69,73],{"class":57,"line":65},2,[55,67,68],{"class":61},"├──",[55,70,72],{"class":71},"sKc5r"," .bench\u002F",[55,74,76],{"class":75},"sOLPB","             # generated types, run logs, materialized workspaces\n",[55,78,80,82,85],{"class":57,"line":79},3,[55,81,68],{"class":61},[55,83,84],{"class":71}," fixtures\u002F",[55,86,87],{"class":75},"           # author-side inputs — never visible to the agent\n",[55,89,91,93,96],{"class":57,"line":90},4,[55,92,68],{"class":61},[55,94,95],{"class":71}," tests\u002F",[55,97,98],{"class":75},"              # the scenarios\n",[55,100,102,105,108],{"class":57,"line":101},5,[55,103,104],{"class":61},"│",[55,106,107],{"class":71},"   └──",[55,109,110],{"class":71}," fix-bug.bench.ts\n",[55,112,114,116,119],{"class":57,"line":113},6,[55,115,68],{"class":61},[55,117,118],{"class":71}," workspace\u002F",[55,120,121],{"class":75},"          # the agent's world — copied fresh for every run\n",[55,123,125,127,130],{"class":57,"line":124},7,[55,126,68],{"class":61},[55,128,129],{"class":71}," bench.config.ts",[55,131,132],{"class":75},"     # what varies, and what is measured\n",[55,134,136,138,141],{"class":57,"line":135},8,[55,137,68],{"class":61},[55,139,140],{"class":71}," package.json",[55,142,143],{"class":75},"        # name, version, npm deps\n",[55,145,147,150,153],{"class":57,"line":146},9,[55,148,149],{"class":61},"└──",[55,151,152],{"class":71}," README.md",[55,154,155],{"class":75},"           # methodology and limitations\n",[41,157,158,159,162,163,166],{},"Two things are required: ",[52,160,161],{},"bench.config.ts"," and at least one file in ",[52,164,165],{},"tests\u002F",". A benchmark\nwith no workspace runs against an empty world, which is fine for anything that does not\nneed one.",[168,169,171],"h2",{"id":170},"the-split-that-matters","The split that matters",[41,173,174],{},[175,176,177,180,181,184],"strong",{},[52,178,179],{},"workspace\u002F"," is what the agent sees. ",[52,182,183],{},"fixtures\u002F"," is what you see.",[41,186,187,188,190],{},"That is a guarantee, not a convention. The workspace is copied into a fresh directory for\nevery single run, and the agent is booted with that copy as its root. Rubrics, expected\noutputs, judge prompts, and setup data live in ",[52,189,183],{},", where the subject cannot reach\nthem.",[41,192,193],{},"If fixtures were visible to the agent, two things would break. A model could read the\nanswer key. And a run would come to depend on the benchmark's own directory layout, which\nmeans it would stop being reproducible the moment someone reorganised the repo.",[168,195,197],{"id":196},"where-each-concern-lives","Where each concern lives",[41,199,200,204,205,209,210,213,214,217],{},[175,201,202],{},[52,203,161],{}," declares what ",[206,207,208],"em",{},"varies"," and what is ",[206,211,212],{},"measured",". The matrix, the trial\ncount, the measurement schema, the workspace policy. Data and one optional ",[52,215,216],{},"setup()"," hook\n— no scenario logic.",[41,219,220,224,225,228,229,232],{},[175,221,222],{},[52,223,165],{}," is what the agent has to ",[206,226,227],{},"do",". Ordinary Bun test files, one scenario each.\nThis is where ",[52,230,231],{},"Axon()"," is booted and where observations are recorded, and it is the file\nyou spend your time in.",[41,234,235],{},"The split is the same one agents and cognets use: identity and behaviour never share a\nfile. The config gives observations stable meaning; the tests produce them.",[168,237,239],{"id":238},"naming","Naming",[41,241,242,243,246,247,250],{},"Test files match ",[52,244,245],{},"tests\u002F**\u002F*.bench.ts"," by default. The ",[52,248,249],{},".bench.ts"," suffix keeps them\ndistinct from the workspace's own test suite — which matters, because a coding benchmark's\nworkspace usually contains real tests the agent is supposed to make pass, and those are\ndata rather than scenarios.",[41,252,253],{},"Override the pattern when a benchmark wants a different layout:",[45,255,259],{"className":256,"code":257,"language":258,"meta":50,"style":50},"language-ts shiki shiki-themes dark-plus","export default defineBench\u003CSchema>({\n    tests: [\"scenarios\u002F**\u002F*.ts\"],\n})\n","ts",[52,260,261,284,299],{"__ignoreMap":50},[55,262,263,267,270,273,277,281],{"class":57,"line":58},[55,264,266],{"class":265},"sYbnZ","export",[55,268,269],{"class":265}," default",[55,271,272],{"class":61}," defineBench",[55,274,276],{"class":275},"sTNBD","\u003C",[55,278,280],{"class":279},"sNl3T","Schema",[55,282,283],{"class":275},">({\n",[55,285,286,290,293,296],{"class":57,"line":65},[55,287,289],{"class":288},"s3F5K","    tests:",[55,291,292],{"class":275}," [",[55,294,295],{"class":71},"\"scenarios\u002F**\u002F*.ts\"",[55,297,298],{"class":275},"],\n",[55,300,301],{"class":57,"line":79},[55,302,303],{"class":275},"})\n",[168,305,307],{"id":306},"what-is-generated","What is generated",[41,309,310,313,314,317],{},[52,311,312],{},".bench\u002F"," is written by ",[52,315,316],{},"axon bench prepare"," and holds the generated declarations, the\nextracted measurement schema, run logs, and any retained workspaces.",[41,319,320],{},"Commit the declarations; ignore the runs. A fresh scaffold does this for you:",[45,322,324],{"className":47,"code":323,"language":49,"meta":50,"style":50},".bench\u002Fruns\u002F\n.bench\u002Fworkspace\u002F\n",[52,325,326,331],{"__ignoreMap":50},[55,327,328],{"class":57,"line":58},[55,329,330],{"class":61},".bench\u002Fruns\u002F\n",[55,332,333],{"class":57,"line":65},[55,334,335],{"class":61},".bench\u002Fworkspace\u002F\n",[168,337,339],{"id":338},"scaffolding-one","Scaffolding one",[45,341,343],{"className":47,"code":342,"language":49,"meta":50,"style":50},"axon bench init my-bench\n",[52,344,345],{"__ignoreMap":50},[55,346,347,350,353,356],{"class":57,"line":58},[55,348,349],{"class":61},"axon",[55,351,352],{"class":71}," bench",[55,354,355],{"class":71}," init",[55,357,358],{"class":71}," my-bench\n",[41,360,361,362,367],{},"Creates the whole structure, ready to prepare. See\n",[363,364,366],"a",{"href":365},"\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Ffirst","Your First Benchmark"," for what to put in it.",[369,370,371],"style",{},"html pre.shiki code .sCudf, html code.shiki .sCudf{--shiki-default:#DCDCAA}html pre.shiki code .sKc5r, html code.shiki .sKc5r{--shiki-default:#CE9178}html pre.shiki code .sOLPB, html code.shiki .sOLPB{--shiki-default:#6A9955}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sYbnZ, html code.shiki .sYbnZ{--shiki-default:#C586C0}html pre.shiki code .sTNBD, html code.shiki .sTNBD{--shiki-default:#D4D4D4}html pre.shiki code .sNl3T, html code.shiki .sNl3T{--shiki-default:#4EC9B0}html pre.shiki code .s3F5K, html code.shiki .s3F5K{--shiki-default:#9CDCFE}",{"title":50,"searchDepth":65,"depth":65,"links":373},[374,375,376,377,378],{"id":170,"depth":65,"text":171},{"id":196,"depth":65,"text":197},{"id":238,"depth":65,"text":239},{"id":306,"depth":65,"text":307},{"id":338,"depth":65,"text":339},"The files that make up a benchmark, and what each one owns.","md",{},"\u002Fdocs\u002Fv2\u002Fbench\u002Ffolder",{"title":32,"description":379},"docs\u002Fv2\u002Fbench\u002Ffolder","tsZaXlMvLKB4yL58_sw5ujpaylN38UG_Z8V_eHD-XBw",1788121883712]