[{"data":1,"prerenderedAt":1019},["ShallowReactive",2],{"i-mdi:magnify":3,"i-simple-icons:visualstudiocode":8,"i-mdi:github":11,"i-mdi:discord":13,"i-mdi:youtube":15,"i-mdi:flask-outline":17,"i-vscode-icons:default-folder-opened":19,"i-vscode-icons:default-folder":22,"i-vscode-icons:folder-type-test":24,"i-vscode-icons:file-type-typescript":26,"i-vscode-icons:file-type-json":28,"docs-\u002Fdocs\u002Fv2\u002Fbench":30},{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":7},0,24,false,"\u003Cpath fill=\"currentColor\" d=\"M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5l-1.5 1.5l-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16A6.5 6.5 0 0 1 3 9.5A6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14S14 12 14 9.5S12 5 9.5 5\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":9,"hidden":10},"\u003Cpath fill=\"currentColor\" d=\"M23.15 2.587L18.21.21a1.49 1.49 0 0 0-1.705.29l-9.46 8.63l-4.12-3.128a1 1 0 0 0-1.276.057L.327 7.261A1 1 0 0 0 .326 8.74L3.899 12L.326 15.26a1 1 0 0 0 .001 1.479L1.65 17.94a1 1 0 0 0 1.276.057l4.12-3.128l9.46 8.63a1.49 1.49 0 0 0 1.704.29l4.942-2.377A1.5 1.5 0 0 0 24 20.06V3.939a1.5 1.5 0 0 0-.85-1.352m-5.146 14.861L10.826 12l7.178-5.448z\"\u002F>",true,{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":12},"\u003Cpath fill=\"currentColor\" d=\"M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5c.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34c-.46-1.16-1.11-1.47-1.11-1.47c-.91-.62.07-.6.07-.6c1 .07 1.53 1.03 1.53 1.03c.87 1.52 2.34 1.07 2.91.83c.09-.65.35-1.09.63-1.34c-2.22-.25-4.55-1.11-4.55-4.92c0-1.11.38-2 1.03-2.71c-.1-.25-.45-1.29.1-2.64c0 0 .84-.27 2.75 1.02c.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02c.55 1.35.2 2.39.1 2.64c.65.71 1.03 1.6 1.03 2.71c0 3.82-2.34 4.66-4.57 4.91c.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":14,"hidden":10},"\u003Cpath fill=\"currentColor\" d=\"m22 24l-5.25-5l.63 2H4.5A2.5 2.5 0 0 1 2 18.5v-15A2.5 2.5 0 0 1 4.5 1h15A2.5 2.5 0 0 1 22 3.5V24M12 6.8c-2.68 0-4.56 1.15-4.56 1.15c1.03-.92 2.83-1.45 2.83-1.45l-.17-.17c-1.69.03-3.22 1.2-3.22 1.2c-1.72 3.59-1.61 6.69-1.61 6.69c1.4 1.81 3.48 1.68 3.48 1.68l.71-.9c-1.25-.27-2.04-1.38-2.04-1.38S9.3 14.9 12 14.9s4.58-1.28 4.58-1.28s-.79 1.11-2.04 1.38l.71.9s2.08.13 3.48-1.68c0 0 .11-3.1-1.61-6.69c0 0-1.53-1.17-3.22-1.2l-.17.17s1.8.53 2.83 1.45c0 0-1.88-1.15-4.56-1.15m-2.07 3.79c.65 0 1.18.57 1.17 1.27c0 .69-.52 1.27-1.17 1.27c-.64 0-1.16-.58-1.16-1.27c0-.7.51-1.27 1.16-1.27m4.17 0c.65 0 1.17.57 1.17 1.27c0 .69-.52 1.27-1.17 1.27c-.64 0-1.16-.58-1.16-1.27c0-.7.51-1.27 1.16-1.27Z\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":16},"\u003Cpath fill=\"currentColor\" d=\"m10 15l5.19-3L10 9zm11.56-7.83c.13.47.22 1.1.28 1.9c.07.8.1 1.49.1 2.09L22 12c0 2.19-.16 3.8-.44 4.83c-.25.9-.83 1.48-1.73 1.73c-.47.13-1.33.22-2.65.28c-1.3.07-2.49.1-3.59.1L12 19c-4.19 0-6.8-.16-7.83-.44c-.9-.25-1.48-.83-1.73-1.73c-.13-.47-.22-1.1-.28-1.9c-.07-.8-.1-1.49-.1-2.09L2 12c0-2.19.16-3.8.44-4.83c.25-.9.83-1.48 1.73-1.73c.47-.13 1.33-.22 2.65-.28c1.3-.07 2.49-.1 3.59-.1L12 5c4.19 0 6.8.16 7.83.44c.9.25 1.48.83 1.73 1.73\"\u002F>",{"left":4,"top":4,"width":5,"height":5,"rotate":4,"vFlip":6,"hFlip":6,"body":18},"\u003Cpath fill=\"currentColor\" d=\"M5 19a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1c0-.21-.07-.41-.18-.57L13 8.35V4h-2v4.35L5.18 18.43c-.11.16-.18.36-.18.57m1 3a3 3 0 0 1-3-3c0-.6.18-1.16.5-1.63L9 7.81V6a1 1 0 0 1-1-1V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v1a1 1 0 0 1-1 1v1.81l5.5 9.56c.32.47.5 1.03.5 1.63a3 3 0 0 1-3 3zm7-6l1.34-1.34L16.27 18H7.73l2.66-4.61zm-.5-4a.5.5 0 0 1 .5.5a.5.5 0 0 1-.5.5a.5.5 0 0 1-.5-.5a.5.5 0 0 1 .5-.5\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":21},32,"\u003Cpath fill=\"#dcb67a\" d=\"M27.4 5.5h-9.2l-2.1 4.2H4.3v16.8h25.2v-21Zm0 18.7H6.6V11.8h20.8Zm0-14.5h-8.2l1-2.1h7.1v2.1Z\"\u002F>\u003Cpath fill=\"#dcb67a\" d=\"M25.7 13.7H.5l3.8 12.8h25.2z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":23},"\u003Cpath fill=\"#c09553\" d=\"M27.5 5.5h-9.3l-2.1 4.2H4.4v16.8h25.2v-21Zm0 4.2h-8.2l1.1-2.1h7.1Z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":25},"\u003Cpath fill=\"#b70b0b\" d=\"M27.5 5.5h-9.3l-2.1 4.2H4.4v16.8h25.2v-21Zm0 4.2h-8.2l1.1-2.1h7.1Z\"\u002F>\u003Cpath fill=\"#fac6c5\" d=\"M28.3 27.423c-1.729-.4-2.332-2.482-1.329-4.583a8.95 8.95 0 0 1 2.217-2.779l.314-.26l.009.357a6.7 6.7 0 0 0 .748 2.425a14 14 0 0 1 .625 1.631a3.2 3.2 0 0 1-.073 1.958a2.67 2.67 0 0 1-1.111 1.087a2.8 2.8 0 0 1-1.4.164m.351-3.565c.124-.143.155-.332.164-1l.011-.825l-.2.158a1.68 1.68 0 0 0-.635 1.4c.07.214.286.448.411.446a.5.5 0 0 0 .249-.179m-15.527 7.087a3.85 3.85 0 0 1-3.031-4.563c.254-1.07.2-1.008 5.726-6.787l5.2-5.442l.738.7l.738.7l-3 3.132l-3 3.132l2.508-.054l2.508-.054l1.773-1.85l1.773-1.85l.735.707l.735.707l-5.213 5.43c-4.783 4.982-5.257 5.452-5.743 5.692a3.9 3.9 0 0 1-2.452.393Zm14.662-12.484c-.138-.082-1.546-1.4-3.129-2.921l-2.879-2.772l-.009-.421a.86.86 0 0 1 .194-.673A.99.99 0 0 1 23 11.3a69 69 0 0 1 6.277 6.023a1 1 0 0 1-1.492 1.133Z\"\u002F>\u003Cpath fill=\"#fac6c5\" d=\"M11.578 26.851a2.1 2.1 0 0 0 2.579 2.614c.191-.2 2.407-2.515 7.387-7.737c-2.505.054-2.475-.055-4.98 0c-5.015 5.195-5.114 5.157-4.986 5.123\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":27},"\u003Cpath fill=\"#007acc\" d=\"M23.827 8.243a4.4 4.4 0 0 1 2.223 1.281a6 6 0 0 1 .852 1.143c.011.045-1.534 1.083-2.471 1.662c-.034.023-.169-.124-.322-.35a2.01 2.01 0 0 0-1.67-1c-1.077-.074-1.771.49-1.766 1.433a1.3 1.3 0 0 0 .153.666c.237.49.677.784 2.059 1.383c2.544 1.095 3.636 1.817 4.31 2.843a5.16 5.16 0 0 1 .416 4.333a4.76 4.76 0 0 1-3.932 2.815a11 11 0 0 1-2.708-.028a6.53 6.53 0 0 1-3.616-1.884a6.3 6.3 0 0 1-.926-1.371a3 3 0 0 1 .327-.208c.158-.09.756-.434 1.32-.761l1.024-.6l.214.312a4.8 4.8 0 0 0 1.35 1.292a3.3 3.3 0 0 0 3.458-.175a1.545 1.545 0 0 0 .2-1.974c-.276-.395-.84-.727-2.443-1.422a8.8 8.8 0 0 1-3.349-2.055a4.7 4.7 0 0 1-.976-1.777a7.1 7.1 0 0 1-.062-2.268a4.33 4.33 0 0 1 3.644-3.374a9 9 0 0 1 2.691.084m-8.343 1.483l.011 1.454h-4.63v13.148H7.6V11.183H2.97V9.755a14 14 0 0 1 .04-1.466c.017-.023 2.832-.034 6.245-.028l6.211.017Z\"\u002F>",{"left":4,"top":4,"width":20,"height":20,"rotate":4,"vFlip":6,"hFlip":6,"body":29},"\u003Cpath fill=\"#f5de19\" d=\"M4.014 14.976a2.5 2.5 0 0 0 1.567-.518a2.38 2.38 0 0 0 .805-1.358a15.3 15.3 0 0 0 .214-2.944q.012-2.085.075-2.747a5.2 5.2 0 0 1 .418-1.686a3 3 0 0 1 .755-1.018A3.05 3.05 0 0 1 9 4.125A6.8 6.8 0 0 1 10.544 4h.7v1.96h-.387a2.34 2.34 0 0 0-1.723.468a3.4 3.4 0 0 0-.425 2.092a36 36 0 0 1-.137 4.133a4.7 4.7 0 0 1-.768 2.06A4.6 4.6 0 0 1 6.1 16a3.8 3.8 0 0 1 1.992 1.754a8.9 8.9 0 0 1 .618 3.865q0 2.435.05 2.9a1.76 1.76 0 0 0 .504 1.181a2.64 2.64 0 0 0 1.592.337h.387V28h-.7a5.7 5.7 0 0 1-1.773-.2a2.97 2.97 0 0 1-1.324-.93a3.35 3.35 0 0 1-.681-1.63a24 24 0 0 1-.165-3.234a16.5 16.5 0 0 0-.214-3.106a2.4 2.4 0 0 0-.805-1.361a2.5 2.5 0 0 0-1.567-.524Zm23.972 2.035a2.5 2.5 0 0 0-1.567.524a2.4 2.4 0 0 0-.805 1.361a16.5 16.5 0 0 0-.212 3.109a24 24 0 0 1-.169 3.234a3.35 3.35 0 0 1-.681 1.63a2.97 2.97 0 0 1-1.324.93a5.7 5.7 0 0 1-1.773.2h-.7V26.04h.387a2.64 2.64 0 0 0 1.592-.337a1.76 1.76 0 0 0 .506-1.186q.05-.462.05-2.9a8.9 8.9 0 0 1 .618-3.865A3.8 3.8 0 0 1 25.9 16a4.6 4.6 0 0 1-1.7-1.286a4.7 4.7 0 0 1-.768-2.06a36 36 0 0 1-.137-4.133a3.4 3.4 0 0 0-.425-2.092a2.34 2.34 0 0 0-1.723-.468h-.387V4h.7a6.8 6.8 0 0 1 1.54.125a3.05 3.05 0 0 1 1.149.581a3 3 0 0 1 .755 1.018a5.2 5.2 0 0 1 .418 1.686q.062.662.075 2.747a15.3 15.3 0 0 0 .212 2.947a2.38 2.38 0 0 0 .805 1.355a2.5 2.5 0 0 0 1.567.518Z\"\u002F>",{"id":31,"title":32,"body":33,"description":1012,"extension":1013,"meta":1014,"navigation":10,"path":1015,"seo":1016,"stem":1017,"__hash__":1018},"docs\u002Fdocs\u002Fv2\u002Fbench\u002Findex.md","Benchmarks",{"type":34,"value":35,"toc":1001},"minimark",[36,40,49,52,57,69,72,76,113,116,192,203,447,630,633,637,640,646,653,657,660,732,749,753,760,763,791,799,803,806,884,910,913,917,920,923,927,937,944,948,957,970,979,988,997],[37,38,32],"h1",{"id":39},"benchmarks",[41,42,43,44,48],"p",{},"\"How good is this agent?\" has never had a real answer. Not because measurement is hard,\nbut because ",[45,46,47],"em",{},"agent"," was never a standardized thing — every framework's agent is a\ndifferent shape, so no result transfers, no comparison holds, and every eval is a\none-off script someone wrote and nobody can rerun.",[41,50,51],{},"Axon agents are a standard shape. That makes them benchmarkable, and it makes benchmarks\nportable, publishable, and comparable — the same as agents and modules.",[53,54,56],"h2",{"id":55},"the-whole-model-in-four-sentences","The whole model in four sentences",[41,58,59,60,64,65,68],{},"Authors declare ",[61,62,63],"strong",{},"what varies"," and ",[61,66,67],{},"how many times to repeat",". The runtime runs every\ntest against every declaration, holding everything else constant. Tests are ordinary Bun\ntests that emit typed observations about what happened. Everything else — duration,\ntokens, cost, failures, coverage — is recorded without you asking.",[41,70,71],{},"That's the system. The rest of this section is consequences.",[53,73,75],{"id":74},"create-one","Create one",[77,78,83],"pre",{"className":79,"code":80,"language":81,"meta":82,"style":82},"language-bash shiki shiki-themes dark-plus","axon bench init my-bench\ncd my-bench\n","bash","",[84,85,86,105],"code",{"__ignoreMap":82},[87,88,91,95,99,102],"span",{"class":89,"line":90},"line",1,[87,92,94],{"class":93},"sCudf","axon",[87,96,98],{"class":97},"sKc5r"," bench",[87,100,101],{"class":97}," init",[87,103,104],{"class":97}," my-bench\n",[87,106,108,111],{"class":89,"line":107},2,[87,109,110],{"class":93},"cd",[87,112,104],{"class":97},[41,114,115],{},"That scaffolds the benchmark, installs the framework, and generates its types:",[77,117,119],{"className":79,"code":118,"language":81,"meta":82,"style":82},"my-bench\u002F\n├── fixtures\u002F              # author-side inputs\n├── tests\u002F\n│   └── example.bench.ts   # a test that emits observations\n├── workspace\u002F             # the tree each run gets a fresh copy of\n├── bench.config.ts        # the matrix: what varies, how many trials\n└── README.md\n",[84,120,121,126,138,146,161,172,183],{"__ignoreMap":82},[87,122,123],{"class":89,"line":90},[87,124,125],{"class":93},"my-bench\u002F\n",[87,127,128,131,134],{"class":89,"line":107},[87,129,130],{"class":93},"├──",[87,132,133],{"class":97}," fixtures\u002F",[87,135,137],{"class":136},"sOLPB","              # author-side inputs\n",[87,139,141,143],{"class":89,"line":140},3,[87,142,130],{"class":93},[87,144,145],{"class":97}," tests\u002F\n",[87,147,149,152,155,158],{"class":89,"line":148},4,[87,150,151],{"class":93},"│",[87,153,154],{"class":97},"   └──",[87,156,157],{"class":97}," example.bench.ts",[87,159,160],{"class":136},"   # a test that emits observations\n",[87,162,164,166,169],{"class":89,"line":163},5,[87,165,130],{"class":93},[87,167,168],{"class":97}," workspace\u002F",[87,170,171],{"class":136},"             # the tree each run gets a fresh copy of\n",[87,173,175,177,180],{"class":89,"line":174},6,[87,176,130],{"class":93},[87,178,179],{"class":97}," bench.config.ts",[87,181,182],{"class":136},"        # the matrix: what varies, how many trials\n",[87,184,186,189],{"class":89,"line":185},7,[87,187,188],{"class":93},"└──",[87,190,191],{"class":97}," README.md\n",[41,193,194,195,198,199,202],{},"Then ",[84,196,197],{},"axon bench run"," executes the matrix, and ",[84,200,201],{},"axon bench result"," reads back what\nhappened. Here's the shape of what you'll be editing:",[77,204,208],{"className":205,"code":206,"language":207,"meta":82,"style":82},"language-ts shiki shiki-themes dark-plus","import { Mock, run } from \"@arcforge\u002Fengines\u002Fmock\"\n\ntype Schema = {\n    \u002F** Has the agent resolved the bug? *\u002F\n    resolved: boolean\n\n    \u002F** Files edited beyond the target. @objective minimize *\u002F\n    collateral: number\n}\n\nexport default defineBench\u003CSchema>({\n    workspace: { source: \".\u002Fworkspace\", retain: \"failed\" },\n\n    matrix: {\n        model: [\n            OpenRouter({ model: \"anthropic\u002Fclaude-sonnet-4.6\" }),\n            OpenRouter({ model: \"openai\u002Fgpt-5\" }),\n        ],\n    },\n\n    trials: 3,\n})\n","ts",[84,209,210,239,244,257,262,273,277,288,299,305,310,331,356,361,370,379,397,411,417,423,428,441],{"__ignoreMap":82},[87,211,212,216,220,224,227,230,233,236],{"class":89,"line":90},[87,213,215],{"class":214},"sYbnZ","import",[87,217,219],{"class":218},"sTNBD"," { ",[87,221,223],{"class":222},"s3F5K","Mock",[87,225,226],{"class":218},", ",[87,228,229],{"class":222},"run",[87,231,232],{"class":218}," } ",[87,234,235],{"class":214},"from",[87,237,238],{"class":97}," \"@arcforge\u002Fengines\u002Fmock\"\n",[87,240,241],{"class":89,"line":107},[87,242,243],{"emptyLinePlaceholder":10},"\n",[87,245,246,250,254],{"class":89,"line":140},[87,247,249],{"class":248},"scz_3","type",[87,251,253],{"class":252},"sNl3T"," Schema",[87,255,256],{"class":218}," = {\n",[87,258,259],{"class":89,"line":148},[87,260,261],{"class":136},"    \u002F** Has the agent resolved the bug? *\u002F\n",[87,263,264,267,270],{"class":89,"line":163},[87,265,266],{"class":222},"    resolved",[87,268,269],{"class":218},": ",[87,271,272],{"class":252},"boolean\n",[87,274,275],{"class":89,"line":174},[87,276,243],{"emptyLinePlaceholder":10},[87,278,279,282,285],{"class":89,"line":185},[87,280,281],{"class":136},"    \u002F** Files edited beyond the target. ",[87,283,284],{"class":248},"@objective",[87,286,287],{"class":136}," minimize *\u002F\n",[87,289,291,294,296],{"class":89,"line":290},8,[87,292,293],{"class":222},"    collateral",[87,295,269],{"class":218},[87,297,298],{"class":252},"number\n",[87,300,302],{"class":89,"line":301},9,[87,303,304],{"class":218},"}\n",[87,306,308],{"class":89,"line":307},10,[87,309,243],{"emptyLinePlaceholder":10},[87,311,313,316,319,322,325,328],{"class":89,"line":312},11,[87,314,315],{"class":214},"export",[87,317,318],{"class":214}," default",[87,320,321],{"class":93}," defineBench",[87,323,324],{"class":218},"\u003C",[87,326,327],{"class":252},"Schema",[87,329,330],{"class":218},">({\n",[87,332,334,337,339,342,345,347,350,353],{"class":89,"line":333},12,[87,335,336],{"class":222},"    workspace:",[87,338,219],{"class":218},[87,340,341],{"class":222},"source:",[87,343,344],{"class":97}," \".\u002Fworkspace\"",[87,346,226],{"class":218},[87,348,349],{"class":222},"retain:",[87,351,352],{"class":97}," \"failed\"",[87,354,355],{"class":218}," },\n",[87,357,359],{"class":89,"line":358},13,[87,360,243],{"emptyLinePlaceholder":10},[87,362,364,367],{"class":89,"line":363},14,[87,365,366],{"class":222},"    matrix:",[87,368,369],{"class":218}," {\n",[87,371,373,376],{"class":89,"line":372},15,[87,374,375],{"class":222},"        model:",[87,377,378],{"class":218}," [\n",[87,380,382,385,388,391,394],{"class":89,"line":381},16,[87,383,384],{"class":93},"            OpenRouter",[87,386,387],{"class":218},"({ ",[87,389,390],{"class":222},"model:",[87,392,393],{"class":97}," \"anthropic\u002Fclaude-sonnet-4.6\"",[87,395,396],{"class":218}," }),\n",[87,398,400,402,404,406,409],{"class":89,"line":399},17,[87,401,384],{"class":93},[87,403,387],{"class":218},[87,405,390],{"class":222},[87,407,408],{"class":97}," \"openai\u002Fgpt-5\"",[87,410,396],{"class":218},[87,412,414],{"class":89,"line":413},18,[87,415,416],{"class":218},"        ],\n",[87,418,420],{"class":89,"line":419},19,[87,421,422],{"class":218},"    },\n",[87,424,426],{"class":89,"line":425},20,[87,427,243],{"emptyLinePlaceholder":10},[87,429,431,434,438],{"class":89,"line":430},21,[87,432,433],{"class":222},"    trials:",[87,435,437],{"class":436},"sDGX8"," 3",[87,439,440],{"class":218},",\n",[87,442,444],{"class":89,"line":443},22,[87,445,446],{"class":218},"})\n",[77,448,450],{"className":205,"code":449,"language":207,"meta":82,"style":82},"\u002F\u002F tests\u002Ffix-bug.bench.ts\nit(\"resolves the failing test\", async ({ workspace }) => {\n    expect(workspace).toHaveFile(\"src\u002Fparser.ts\")\n\n    const { axon } = await Axon()\n    await axon.request(\"Fix the failing test in src\u002Fparser.ts\")\n\n    observe(\"resolved\", await workspace.tests.pass())\n    observe(\"collateral\", (await workspace.changed()).length - 1)\n})\n",[84,451,452,457,487,510,514,536,557,561,591,626],{"__ignoreMap":82},[87,453,454],{"class":89,"line":90},[87,455,456],{"class":136},"\u002F\u002F tests\u002Ffix-bug.bench.ts\n",[87,458,459,462,465,468,470,473,476,479,482,485],{"class":89,"line":107},[87,460,461],{"class":93},"it",[87,463,464],{"class":218},"(",[87,466,467],{"class":97},"\"resolves the failing test\"",[87,469,226],{"class":218},[87,471,472],{"class":248},"async",[87,474,475],{"class":218}," ({ ",[87,477,478],{"class":222},"workspace",[87,480,481],{"class":218}," }) ",[87,483,484],{"class":248},"=>",[87,486,369],{"class":218},[87,488,489,492,494,496,499,502,504,507],{"class":89,"line":140},[87,490,491],{"class":93},"    expect",[87,493,464],{"class":218},[87,495,478],{"class":222},[87,497,498],{"class":218},").",[87,500,501],{"class":93},"toHaveFile",[87,503,464],{"class":218},[87,505,506],{"class":97},"\"src\u002Fparser.ts\"",[87,508,509],{"class":218},")\n",[87,511,512],{"class":89,"line":148},[87,513,243],{"emptyLinePlaceholder":10},[87,515,516,519,521,524,527,530,533],{"class":89,"line":163},[87,517,518],{"class":248},"    const",[87,520,219],{"class":218},[87,522,94],{"class":523},"s9McN",[87,525,526],{"class":218}," } = ",[87,528,529],{"class":214},"await",[87,531,532],{"class":93}," Axon",[87,534,535],{"class":218},"()\n",[87,537,538,541,544,547,550,552,555],{"class":89,"line":174},[87,539,540],{"class":214},"    await",[87,542,543],{"class":222}," axon",[87,545,546],{"class":218},".",[87,548,549],{"class":93},"request",[87,551,464],{"class":218},[87,553,554],{"class":97},"\"Fix the failing test in src\u002Fparser.ts\"",[87,556,509],{"class":218},[87,558,559],{"class":89,"line":185},[87,560,243],{"emptyLinePlaceholder":10},[87,562,563,566,568,571,573,575,578,580,583,585,588],{"class":89,"line":290},[87,564,565],{"class":93},"    observe",[87,567,464],{"class":218},[87,569,570],{"class":97},"\"resolved\"",[87,572,226],{"class":218},[87,574,529],{"class":214},[87,576,577],{"class":222}," workspace",[87,579,546],{"class":218},[87,581,582],{"class":222},"tests",[87,584,546],{"class":218},[87,586,587],{"class":93},"pass",[87,589,590],{"class":218},"())\n",[87,592,593,595,597,600,603,605,607,609,612,615,618,621,624],{"class":89,"line":301},[87,594,565],{"class":93},[87,596,464],{"class":218},[87,598,599],{"class":97},"\"collateral\"",[87,601,602],{"class":218},", (",[87,604,529],{"class":214},[87,606,577],{"class":222},[87,608,546],{"class":218},[87,610,611],{"class":93},"changed",[87,613,614],{"class":218},"()).",[87,616,617],{"class":222},"length",[87,619,620],{"class":218}," - ",[87,622,623],{"class":436},"1",[87,625,509],{"class":218},[87,627,628],{"class":89,"line":307},[87,629,446],{"class":218},[41,631,632],{},"Two models, three trials each, one scenario: six runs, twelve observations, and a full\nsession log for every one of them.",[53,634,636],{"id":635},"a-benchmark-is-a-controlled-experiment","A benchmark is a controlled experiment",[41,638,639],{},"The idea is old-fashioned science: hold everything constant, vary one thing, measure what\nchanges.",[41,641,642,645],{},[84,643,644],{},"matrix"," is that one thing. Every key is a binding point in the agent — the model, the\ncognet, the whole agent, any path into its config — and every value is a variation to\ntry. One key is a controlled experiment. Several keys multiply into a grid, which is\nsometimes what you want and always a decision you made rather than one the runtime made\nfor you.",[41,647,648,649,652],{},"The harness is the same ",[84,650,651],{},"Axon()"," runtime used everywhere else. Each cell applies its\nvariation to the normal boot, so the thing you measure is the thing that runs in\nproduction.",[53,654,656],{"id":655},"a-benchmark-is-a-folder","A benchmark is a folder",[41,658,659],{},"Like everything else in Axon:",[77,661,663],{"className":79,"code":662,"language":81,"meta":82,"style":82},"my-bench\u002F\n├── .bench\u002F             # generated types, run logs, materialized workspaces\n├── fixtures\u002F           # author-side inputs — never visible to the agent\n├── tests\u002F              # the scenarios\n│   └── fix-bug.bench.ts\n├── workspace\u002F          # the agent's world — copied fresh for every run\n├── bench.config.ts     # what varies, and what is measured\n└── package.json\n",[84,664,665,669,679,688,698,707,716,725],{"__ignoreMap":82},[87,666,667],{"class":89,"line":90},[87,668,125],{"class":93},[87,670,671,673,676],{"class":89,"line":107},[87,672,130],{"class":93},[87,674,675],{"class":97}," .bench\u002F",[87,677,678],{"class":136},"             # generated types, run logs, materialized workspaces\n",[87,680,681,683,685],{"class":89,"line":140},[87,682,130],{"class":93},[87,684,133],{"class":97},[87,686,687],{"class":136},"           # author-side inputs — never visible to the agent\n",[87,689,690,692,695],{"class":89,"line":148},[87,691,130],{"class":93},[87,693,694],{"class":97}," tests\u002F",[87,696,697],{"class":136},"              # the scenarios\n",[87,699,700,702,704],{"class":89,"line":163},[87,701,151],{"class":93},[87,703,154],{"class":97},[87,705,706],{"class":97}," fix-bug.bench.ts\n",[87,708,709,711,713],{"class":89,"line":174},[87,710,130],{"class":93},[87,712,168],{"class":97},[87,714,715],{"class":136},"          # the agent's world — copied fresh for every run\n",[87,717,718,720,722],{"class":89,"line":185},[87,719,130],{"class":93},[87,721,179],{"class":97},[87,723,724],{"class":136},"     # what varies, and what is measured\n",[87,726,727,729],{"class":89,"line":290},[87,728,188],{"class":93},[87,730,731],{"class":97}," package.json\n",[41,733,734,735,738,739,742,743,745,746,748],{},"Each directory means exactly one thing, and the ",[84,736,737],{},"workspace\u002F"," ↔ ",[84,740,741],{},"fixtures\u002F"," split is a\nguarantee rather than a convention. The agent sees ",[84,744,737],{}," and nothing else. Rubrics,\nexpected outputs, and setup data live in ",[84,747,741],{},", where the subject cannot read them\nand a run cannot come to depend on the benchmark's own layout.",[53,750,752],{"id":751},"tests-measure-they-do-not-judge","Tests measure, they do not judge",[41,754,755,756,759],{},"A test suite says ",[45,757,758],{},"these assertions must hold or I am unhappy",". A benchmark cannot say\nthat, because it exists precisely when you do not know what will happen. Halting on the\nfirst surprise is the opposite of what you want.",[41,761,762],{},"So benchmarks have two verbs, and the difference is intent:",[764,765,766,779],"ul",{},[767,768,769,774,775,778],"li",{},[61,770,771],{},[84,772,773],{},"expect"," — did the ",[45,776,777],{},"scenario"," hold? The workspace copied, the agent booted, the\nfile the task refers to exists. A failure here means this run measured nothing, and it\nis excluded rather than scored zero.",[767,780,781,786,787,790],{},[61,782,783],{},[84,784,785],{},"observe"," — how did the ",[45,788,789],{},"subject"," do? Recorded whatever the value is. A model that\nfails every trial is the finding, not a broken benchmark.",[41,792,793,794,546],{},"That single rule is most of what separates writing a benchmark from writing tests. See\n",[795,796,798],"a",{"href":797},"\u002Fdocs\u002Fv2\u002Fbench\u002Fmeasuring","Measuring",[53,800,802],{"id":801},"what-you-get-without-asking-for-it","What you get without asking for it",[41,804,805],{},"Every trial records its own physics, derived from the session log rather than declared:",[807,808,809,820],"table",{},[810,811,812],"thead",{},[813,814,815,818],"tr",{},[816,817],"th",{},[816,819],{},[821,822,823,834,844,854,864,874],"tbody",{},[813,824,825,831],{},[826,827,828],"td",{},[84,829,830],{},"durationMs",[826,832,833],{},"wall clock for the trial",[813,835,836,841],{},[826,837,838],{},[84,839,840],{},"tokens",[826,842,843],{},"input and output, summed across every engine call",[813,845,846,851],{},[826,847,848],{},[84,849,850],{},"costUsd",[826,852,853],{},"real spend, when the provider prices the call",[813,855,856,861],{},[826,857,858],{},[84,859,860],{},"engineCalls",[826,862,863],{},"how many times the model was consulted",[813,865,866,871],{},[826,867,868],{},[84,869,870],{},"toolCalls",[826,872,873],{},"how many times it reached for the world",[813,875,876,881],{},[826,877,878],{},[84,879,880],{},"errors",[826,882,883],{},"engine and run failures",[41,885,886,887,226,890,226,893,226,896,226,899,226,902,905,906,909],{},"Alongside faults (typed: ",[84,888,889],{},"boot",[84,891,892],{},"timeout",[84,894,895],{},"budget",[84,897,898],{},"cancelled",[84,900,901],{},"process",[84,903,904],{},"protocol","),\nthe full session for every trial, and coverage — expected versus executed versus\ncompleted, and ",[45,907,908],{},"why"," anything is missing.",[41,911,912],{},"That last one matters more than it sounds. Most published benchmark numbers cannot\ndistinguish \"scored zero\" from \"never ran\". Here they are different facts, and both are in\nthe record.",[53,914,916],{"id":915},"results-are-evidence-not-summaries","Results are evidence, not summaries",[41,918,919],{},"A run executes, persists events, and derives its result by replaying that log. The result\nholds trials, observations, artifacts, sessions, faults, and coverage — and deliberately\nholds no aggregates or charts.",[41,921,922],{},"Both are projections over the record. Storing them would mean storing a claim next to the\nevidence for it, and the two can drift. Deriving them means a result from six months ago\ncan answer a question nobody had asked yet.",[53,924,926],{"id":925},"benchmarks-are-registry-artifacts","Benchmarks are registry artifacts",[41,928,929,932,933,936],{},[84,930,931],{},"axon publish"," ships a benchmark the same way it ships an agent or a module, and\n",[84,934,935],{},"axon clone"," brings it down to run against your own work.",[41,938,939,940,943],{},"What ships is the ",[45,941,942],{},"definition"," — config, tests, workspace. Results are not part of the\nartifact: a run's observations belong to whoever spent the tokens producing them, and\npublishing a benchmark is a different act from publishing a score.",[53,945,947],{"id":946},"start-here","Start here",[41,949,950,956],{},[61,951,952],{},[795,953,955],{"href":954},"\u002Fdocs\u002Fv2\u002Fbench\u002Fmatrix","The Matrix"," — binding points, dotted paths, and why one\nvariable is the default rather than a limitation.",[41,958,959,963,964,966,967,969],{},[61,960,961],{},[795,962,798],{"href":797}," — ",[84,965,773],{}," versus ",[84,968,785],{},", and declaring a\nschema as a TypeScript type.",[41,971,972,978],{},[61,973,974],{},[795,975,977],{"href":976},"\u002Fdocs\u002Fv2\u002Fbench\u002Fresults","Results"," — what a run produces, and why aggregates are\nderived rather than stored.",[41,980,981,987],{},[61,982,983],{},[795,984,986],{"href":985},"\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Ffirst","Your First Benchmark"," — a real bug, two models, end\nto end.",[41,989,990,996],{},[61,991,992],{},[795,993,995],{"href":994},"\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Fmock","Testing with Mock"," — exercise the whole flow with no\nAPI keys and no spend.",[998,999,1000],"style",{},"html pre.shiki code .sCudf, html code.shiki .sCudf{--shiki-default:#DCDCAA}html pre.shiki code .sKc5r, html code.shiki .sKc5r{--shiki-default:#CE9178}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sOLPB, html code.shiki .sOLPB{--shiki-default:#6A9955}html pre.shiki code .sYbnZ, html code.shiki .sYbnZ{--shiki-default:#C586C0}html pre.shiki code .sTNBD, html code.shiki .sTNBD{--shiki-default:#D4D4D4}html pre.shiki code .s3F5K, html code.shiki .s3F5K{--shiki-default:#9CDCFE}html pre.shiki code .scz_3, html code.shiki .scz_3{--shiki-default:#569CD6}html pre.shiki code .sNl3T, html code.shiki .sNl3T{--shiki-default:#4EC9B0}html pre.shiki code .sDGX8, html code.shiki .sDGX8{--shiki-default:#B5CEA8}html pre.shiki code .s9McN, html code.shiki .s9McN{--shiki-default:#4FC1FF}",{"title":82,"searchDepth":107,"depth":107,"links":1002},[1003,1004,1005,1006,1007,1008,1009,1010,1011],{"id":55,"depth":107,"text":56},{"id":74,"depth":107,"text":75},{"id":635,"depth":107,"text":636},{"id":655,"depth":107,"text":656},{"id":751,"depth":107,"text":752},{"id":801,"depth":107,"text":802},{"id":915,"depth":107,"text":916},{"id":925,"depth":107,"text":926},{"id":946,"depth":107,"text":947},"Controlled experiments over agents — change one variable, hold everything else, measure what moves.","md",{},"\u002Fdocs\u002Fv2\u002Fbench",{"title":32,"description":1012},"docs\u002Fv2\u002Fbench\u002Findex","jhYs21IObRK0-s086953xCcpiGSVN6sy_kVw6ILL_4o",1788121883719]