[{"data":1,"prerenderedAt":577},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Fmock":3},{"id":4,"title":5,"body":6,"description":570,"extension":571,"meta":572,"navigation":71,"path":573,"seo":574,"stem":575,"__hash__":576},"docs\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Fmock.md","Testing with Mock",{"type":7,"value":8,"toc":563},"minimark",[9,13,17,24,144,151,156,169,266,269,272,276,415,425,429,435,462,465,469,472,535,542,546,549,556,559],[10,11,5],"h1",{"id":12},"testing-with-mock",[14,15,16],"p",{},"A benchmark is code, and code has bugs. The problem is that finding them normally costs\nmoney: every run of a two-model, three-trial matrix is six real agent invocations, and the\nfirst five attempts are usually you discovering that a measurement id was wrong.",[14,18,19,23],{},[20,21,22],"code",{},"Mock()"," is a real engine that never calls a provider.",[25,26,31],"pre",{"className":27,"code":28,"language":29,"meta":30,"style":30},"language-ts shiki shiki-themes dark-plus","import { Mock, run } from \"@arcforge\u002Fengines\u002Fmock\"\n\nmatrix: {\n    model: [\n        Mock({ fix: run(\"export function parseLine(line: string) { \u002F* correct *\u002F }\") }),\n        Mock({ fix: \"I can't do that\" }),\n    ],\n}\n","ts","",[20,32,33,66,73,83,92,117,132,138],{"__ignoreMap":30},[34,35,38,42,46,50,53,56,59,62],"span",{"class":36,"line":37},"line",1,[34,39,41],{"class":40},"sYbnZ","import",[34,43,45],{"class":44},"sTNBD"," { ",[34,47,49],{"class":48},"s3F5K","Mock",[34,51,52],{"class":44},", ",[34,54,55],{"class":48},"run",[34,57,58],{"class":44}," } ",[34,60,61],{"class":40},"from",[34,63,65],{"class":64},"sKc5r"," \"@arcforge\u002Fengines\u002Fmock\"\n",[34,67,69],{"class":36,"line":68},2,[34,70,72],{"emptyLinePlaceholder":71},true,"\n",[34,74,76,80],{"class":36,"line":75},3,[34,77,79],{"class":78},"sqDPy","matrix",[34,81,82],{"class":44},": {\n",[34,84,86,89],{"class":36,"line":85},4,[34,87,88],{"class":78},"    model",[34,90,91],{"class":44},": [\n",[34,93,95,99,102,105,108,111,114],{"class":36,"line":94},5,[34,96,98],{"class":97},"sCudf","        Mock",[34,100,101],{"class":44},"({ ",[34,103,104],{"class":48},"fix:",[34,106,107],{"class":97}," run",[34,109,110],{"class":44},"(",[34,112,113],{"class":64},"\"export function parseLine(line: string) { \u002F* correct *\u002F }\"",[34,115,116],{"class":44},") }),\n",[34,118,120,122,124,126,129],{"class":36,"line":119},6,[34,121,98],{"class":97},[34,123,101],{"class":44},[34,125,104],{"class":48},[34,127,128],{"class":64}," \"I can't do that\"",[34,130,131],{"class":44}," }),\n",[34,133,135],{"class":36,"line":134},7,[34,136,137],{"class":44},"    ],\n",[34,139,141],{"class":36,"line":140},8,[34,142,143],{"class":44},"}\n",[14,145,146,147,150],{},"Two cells, six runs with ",[20,148,149],{},"trials: 3",", and the full pipeline exercised: matrix expansion,\nsubject binding, workspace copying, measurement collection, replay. Zero cost.",[152,153,155],"h2",{"id":154},"mocks-vary-behaviour-not-labels","Mocks vary behaviour, not labels",[14,157,158,159,163,164,168],{},"This matters more than it first appears. A mock that always says the same thing tests that\nthe harness runs; a mock whose ",[160,161,162],"em",{},"behaviour"," differs per cell tests that the harness\n",[165,166,167],"strong",{},"measures",".",[25,170,172],{"className":27,"code":171,"language":29,"meta":30,"style":30},"matrix: {\n    model: [\n        Mock({ fix: run(\"...correct patch...\") }),               \u002F\u002F should score 1\n        Mock({ fix: \"I can't do that\" }),                        \u002F\u002F should score 0\n        Mock({ fix: [run(\"...wrong...\"), run(\"...right...\")] }), \u002F\u002F succeeds on retry\n    ],\n}\n",[20,173,174,180,186,208,224,258,262],{"__ignoreMap":30},[34,175,176,178],{"class":36,"line":37},[34,177,79],{"class":78},[34,179,82],{"class":44},[34,181,182,184],{"class":36,"line":68},[34,183,88],{"class":78},[34,185,91],{"class":44},[34,187,188,190,192,194,196,198,201,204],{"class":36,"line":75},[34,189,98],{"class":97},[34,191,101],{"class":44},[34,193,104],{"class":48},[34,195,107],{"class":97},[34,197,110],{"class":44},[34,199,200],{"class":64},"\"...correct patch...\"",[34,202,203],{"class":44},") }),               ",[34,205,207],{"class":206},"sOLPB","\u002F\u002F should score 1\n",[34,209,210,212,214,216,218,221],{"class":36,"line":85},[34,211,98],{"class":97},[34,213,101],{"class":44},[34,215,104],{"class":48},[34,217,128],{"class":64},[34,219,220],{"class":44}," }),                        ",[34,222,223],{"class":206},"\u002F\u002F should score 0\n",[34,225,226,228,230,232,235,237,239,242,245,247,249,252,255],{"class":36,"line":94},[34,227,98],{"class":97},[34,229,101],{"class":44},[34,231,104],{"class":48},[34,233,234],{"class":44}," [",[34,236,55],{"class":97},[34,238,110],{"class":44},[34,240,241],{"class":64},"\"...wrong...\"",[34,243,244],{"class":44},"), ",[34,246,55],{"class":97},[34,248,110],{"class":44},[34,250,251],{"class":64},"\"...right...\"",[34,253,254],{"class":44},")] }), ",[34,256,257],{"class":206},"\u002F\u002F succeeds on retry\n",[34,259,260],{"class":36,"line":119},[34,261,137],{"class":44},[34,263,264],{"class":36,"line":134},[34,265,143],{"class":44},[14,267,268],{},"Three cells with three known outcomes. Run it, and the result should say exactly that. If\nit does not, the bug is in your benchmark rather than in any model — which is precisely the\nthing that is expensive to discover with real inference.",[14,270,271],{},"The third case is worth noting: an array is an ordered sequence, one step per matching\ncall. It is how you test that a benchmark handles an agent that gets there on the second\nattempt.",[152,273,275],{"id":274},"what-a-mock-can-do","What a mock can do",[25,277,279],{"className":27,"code":278,"language":29,"meta":30,"style":30},"Mock()                                   \u002F\u002F echoes the user's message\nMock({ hello: \"Hi there!\" })             \u002F\u002F substring match on the last user message\nMock({ fix: run(\"1 + 1\") })              \u002F\u002F execute code in the capsule\nMock({ fix: [run(\"...\"), \"done\"] })      \u002F\u002F ordered sequence\nMock((req) => `You said: ${req.messages.at(-1)?.content}`)\n",[20,280,281,291,309,330,358],{"__ignoreMap":30},[34,282,283,285,288],{"class":36,"line":37},[34,284,49],{"class":97},[34,286,287],{"class":44},"()                                   ",[34,289,290],{"class":206},"\u002F\u002F echoes the user's message\n",[34,292,293,295,297,300,303,306],{"class":36,"line":68},[34,294,49],{"class":97},[34,296,101],{"class":44},[34,298,299],{"class":48},"hello:",[34,301,302],{"class":64}," \"Hi there!\"",[34,304,305],{"class":44}," })             ",[34,307,308],{"class":206},"\u002F\u002F substring match on the last user message\n",[34,310,311,313,315,317,319,321,324,327],{"class":36,"line":75},[34,312,49],{"class":97},[34,314,101],{"class":44},[34,316,104],{"class":48},[34,318,107],{"class":97},[34,320,110],{"class":44},[34,322,323],{"class":64},"\"1 + 1\"",[34,325,326],{"class":44},") })              ",[34,328,329],{"class":206},"\u002F\u002F execute code in the capsule\n",[34,331,332,334,336,338,340,342,344,347,349,352,355],{"class":36,"line":85},[34,333,49],{"class":97},[34,335,101],{"class":44},[34,337,104],{"class":48},[34,339,234],{"class":44},[34,341,55],{"class":97},[34,343,110],{"class":44},[34,345,346],{"class":64},"\"...\"",[34,348,244],{"class":44},[34,350,351],{"class":64},"\"done\"",[34,353,354],{"class":44},"] })      ",[34,356,357],{"class":206},"\u002F\u002F ordered sequence\n",[34,359,360,362,365,368,371,375,378,381,383,385,388,390,393,396,400,403,406,409,412],{"class":36,"line":94},[34,361,49],{"class":97},[34,363,364],{"class":44},"((",[34,366,367],{"class":48},"req",[34,369,370],{"class":44},") ",[34,372,374],{"class":373},"scz_3","=>",[34,376,377],{"class":64}," `You said: ",[34,379,380],{"class":373},"${",[34,382,367],{"class":48},[34,384,168],{"class":44},[34,386,387],{"class":48},"messages",[34,389,168],{"class":44},[34,391,392],{"class":97},"at",[34,394,395],{"class":44},"(-",[34,397,399],{"class":398},"sDGX8","1",[34,401,402],{"class":44},")?.",[34,404,405],{"class":48},"content",[34,407,408],{"class":373},"}",[34,410,411],{"class":64},"`",[34,413,414],{"class":44},")\n",[14,416,417,420,421,424],{},[20,418,419],{},"run()"," executes in the agent's capsule under the same policy as any other engine, which is\nwhat makes a mock able to actually modify the workspace — and therefore able to make\n",[20,422,423],{},"workspace.tests.pass()"," return true.",[152,426,428],{"id":427},"distinguishing-identical-mocks","Distinguishing identical mocks",[14,430,431,432,434],{},"Cell ids come from the variation's value, and every ",[20,433,22],{}," derives the same name. When\nthat happens they are suffixed rather than rejected:",[25,436,440],{"className":437,"code":438,"language":439,"meta":30,"style":30},"language-bash shiki shiki-themes dark-plus","model=mock-1\nmodel=mock-2\n","bash",[20,441,442,453],{"__ignoreMap":30},[34,443,444,447,450],{"class":36,"line":37},[34,445,446],{"class":48},"model",[34,448,449],{"class":44},"=",[34,451,452],{"class":64},"mock-1\n",[34,454,455,457,459],{"class":36,"line":68},[34,456,446],{"class":48},[34,458,449],{"class":44},[34,460,461],{"class":64},"mock-2\n",[14,463,464],{},"Two mocks with different reply maps are a meaningful pair even though their identity cannot\nexpress the difference. Rejecting the config would be the wrong call; silently merging them\nwould be worse.",[152,466,468],{"id":467},"keep-the-real-matrix-nearby","Keep the real matrix nearby",[14,470,471],{},"The usual pattern is to develop against mocks and switch to real engines once the benchmark\nitself is proven:",[25,473,475],{"className":27,"code":474,"language":29,"meta":30,"style":30},"matrix: {\n    model: [\n        \u002F\u002F Mock({ fix: run(\"...\") }),\n        \u002F\u002F Mock({ fix: \"I can't do that\" }),\n        OpenRouter({ model: \"anthropic\u002Fclaude-sonnet-4.6\" }),\n        OpenRouter({ model: \"openai\u002Fgpt-5\" }),\n    ],\n}\n",[20,476,477,483,489,494,499,514,527,531],{"__ignoreMap":30},[34,478,479,481],{"class":36,"line":37},[34,480,79],{"class":78},[34,482,82],{"class":44},[34,484,485,487],{"class":36,"line":68},[34,486,88],{"class":78},[34,488,91],{"class":44},[34,490,491],{"class":36,"line":75},[34,492,493],{"class":206},"        \u002F\u002F Mock({ fix: run(\"...\") }),\n",[34,495,496],{"class":36,"line":85},[34,497,498],{"class":206},"        \u002F\u002F Mock({ fix: \"I can't do that\" }),\n",[34,500,501,504,506,509,512],{"class":36,"line":94},[34,502,503],{"class":97},"        OpenRouter",[34,505,101],{"class":44},[34,507,508],{"class":48},"model:",[34,510,511],{"class":64}," \"anthropic\u002Fclaude-sonnet-4.6\"",[34,513,131],{"class":44},[34,515,516,518,520,522,525],{"class":36,"line":119},[34,517,503],{"class":97},[34,519,101],{"class":44},[34,521,508],{"class":48},[34,523,524],{"class":64}," \"openai\u002Fgpt-5\"",[34,526,131],{"class":44},[34,528,529],{"class":36,"line":134},[34,530,137],{"class":44},[34,532,533],{"class":36,"line":140},[34,534,143],{"class":44},[14,536,537,538,541],{},"Nothing else in the benchmark changes. The scenario, the measurements, the workspace and\nthe assertions are identical — only what sits behind ",[20,539,540],{},"Axon()"," differs, which is the entire\npoint of the matrix being a binding point rather than a mode.",[152,543,545],{"id":544},"what-mocks-cannot-tell-you","What mocks cannot tell you",[14,547,548],{},"Whether the task is well-specified. A mock does exactly what you told it to, so a prompt\nthat would be ambiguous to a real model looks fine.",[14,550,551,552,555],{},"Whether the measurements are meaningful. Mocks confirm the pipeline records what happened;\nonly real runs reveal that ",[20,553,554],{},"resolved"," was measuring something narrower than you meant.",[14,557,558],{},"Use them to prove the benchmark works. Use real engines to find out whether it asks a good\nquestion.",[560,561,562],"style",{},"html pre.shiki code .sYbnZ, html code.shiki .sYbnZ{--shiki-default:#C586C0}html pre.shiki code .sTNBD, html code.shiki .sTNBD{--shiki-default:#D4D4D4}html pre.shiki code .s3F5K, html code.shiki .s3F5K{--shiki-default:#9CDCFE}html pre.shiki code .sKc5r, html code.shiki .sKc5r{--shiki-default:#CE9178}html pre.shiki code .sqDPy, html code.shiki .sqDPy{--shiki-default:#C8C8C8}html pre.shiki code .sCudf, html code.shiki .sCudf{--shiki-default:#DCDCAA}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sOLPB, html code.shiki .sOLPB{--shiki-default:#6A9955}html pre.shiki code .scz_3, html code.shiki .scz_3{--shiki-default:#569CD6}html pre.shiki code .sDGX8, html code.shiki .sDGX8{--shiki-default:#B5CEA8}",{"title":30,"searchDepth":68,"depth":68,"links":564},[565,566,567,568,569],{"id":154,"depth":68,"text":155},{"id":274,"depth":68,"text":275},{"id":427,"depth":68,"text":428},{"id":467,"depth":68,"text":468},{"id":544,"depth":68,"text":545},"Exercise the whole benchmark flow with no API keys and no spend.","md",{},"\u002Fdocs\u002Fv2\u002Fbench\u002Fbuilding\u002Fmock",{"title":5,"description":570},"docs\u002Fv2\u002Fbench\u002Fbuilding\u002Fmock","vZzitobjkyzuiQ-vINqcpgP-cSMywDngx4Nd7RbJmu0",1785237008160]