[{"data":1,"prerenderedAt":505},["ShallowReactive",2],{"docs-\u002Fdocs\u002Fv2\u002Fbench\u002Ffixtures":3},{"id":4,"title":5,"body":6,"description":498,"extension":499,"meta":500,"navigation":320,"path":501,"seo":502,"stem":503,"__hash__":504},"docs\u002Fdocs\u002Fv2\u002Fbench\u002Ffixtures.md","fixtures\u002F",{"type":7,"value":8,"toc":492},"minimark",[9,13,29,105,112,117,120,127,133,137,143,233,236,420,423,427,430,464,471,475,485,488],[10,11,5],"h1",{"id":12},"fixtures",[14,15,16,19,20,24,25,28],"p",{},[17,18,5],"code",{}," is for everything the ",[21,22,23],"em",{},"benchmark"," needs and the ",[21,26,27],{},"agent"," must not have.",[30,31,36],"pre",{"className":32,"code":33,"language":34,"meta":35,"style":35},"language-bash shiki shiki-themes dark-plus","my-bench\u002F\n└── fixtures\u002F\n    ├── expected\u002F\n    │   └── parser.ts       # the reference solution\n    ├── rubric.md           # grading criteria for a judge\n    └── seed.sql            # data for setup()\n","bash","",[17,37,38,47,57,66,82,93],{"__ignoreMap":35},[39,40,43],"span",{"class":41,"line":42},"line",1,[39,44,46],{"class":45},"sCudf","my-bench\u002F\n",[39,48,50,53],{"class":41,"line":49},2,[39,51,52],{"class":45},"└──",[39,54,56],{"class":55},"sKc5r"," fixtures\u002F\n",[39,58,60,63],{"class":41,"line":59},3,[39,61,62],{"class":45},"    ├──",[39,64,65],{"class":55}," expected\u002F\n",[39,67,69,72,75,78],{"class":41,"line":68},4,[39,70,71],{"class":45},"    │",[39,73,74],{"class":55},"   └──",[39,76,77],{"class":55}," parser.ts",[39,79,81],{"class":80},"sOLPB","       # the reference solution\n",[39,83,85,87,90],{"class":41,"line":84},5,[39,86,62],{"class":45},[39,88,89],{"class":55}," rubric.md",[39,91,92],{"class":80},"           # grading criteria for a judge\n",[39,94,96,99,102],{"class":41,"line":95},6,[39,97,98],{"class":45},"    └──",[39,100,101],{"class":55}," seed.sql",[39,103,104],{"class":80},"            # data for setup()\n",[14,106,107,108,111],{},"Expected outputs, grading rubrics, judge prompts, seed data, comparison baselines. Read by\n",[17,109,110],{},"setup()"," and by your tests; never copied into the world the agent runs in.",[113,114,116],"h2",{"id":115},"why-it-is-a-separate-directory","Why it is a separate directory",[14,118,119],{},"Two reasons, and both are about keeping results honest.",[14,121,122,126],{},[123,124,125],"strong",{},"The agent could read the answer."," A reference solution sitting in the workspace is\nfindable by anything with a filesystem tool. A benchmark that leaks its own answer key is\nmeasuring retrieval, not capability — and it would do so silently, producing high scores\nthat look like good news.",[14,128,129,132],{},[123,130,131],{},"Reproducibility."," The workspace is copied and the copy is the agent's root, so its\nreachable filesystem is bounded and identical every run. Fixtures are read from the\nbenchmark directory by the harness, which means they cannot become part of what the subject\ndepends on. Move the benchmark, restructure the repo, clone it onto another machine — the\nagent's world is unchanged.",[113,134,136],{"id":135},"using-them","Using them",[14,138,139,140,142],{},"From ",[17,141,110],{},", which runs after the workspace copy and before the agent boots:",[30,144,148],{"className":145,"code":146,"language":147,"meta":35,"style":35},"language-ts shiki shiki-themes dark-plus","export default defineBench\u003CSchema>({\n    async setup() {\n        await seedDatabase(await Bun.file(\".\u002Ffixtures\u002Fseed.sql\").text())\n    },\n})\n","ts",[17,149,150,173,185,223,228],{"__ignoreMap":35},[39,151,152,156,159,162,166,170],{"class":41,"line":42},[39,153,155],{"class":154},"sYbnZ","export",[39,157,158],{"class":154}," default",[39,160,161],{"class":45}," defineBench",[39,163,165],{"class":164},"sTNBD","\u003C",[39,167,169],{"class":168},"sNl3T","Schema",[39,171,172],{"class":164},">({\n",[39,174,175,179,182],{"class":41,"line":49},[39,176,178],{"class":177},"scz_3","    async",[39,180,181],{"class":45}," setup",[39,183,184],{"class":164},"() {\n",[39,186,187,190,193,196,199,203,206,209,211,214,217,220],{"class":41,"line":59},[39,188,189],{"class":154},"        await",[39,191,192],{"class":45}," seedDatabase",[39,194,195],{"class":164},"(",[39,197,198],{"class":154},"await",[39,200,202],{"class":201},"s3F5K"," Bun",[39,204,205],{"class":164},".",[39,207,208],{"class":45},"file",[39,210,195],{"class":164},[39,212,213],{"class":55},"\".\u002Ffixtures\u002Fseed.sql\"",[39,215,216],{"class":164},").",[39,218,219],{"class":45},"text",[39,221,222],{"class":164},"())\n",[39,224,225],{"class":41,"line":68},[39,226,227],{"class":164},"    },\n",[39,229,230],{"class":41,"line":84},[39,231,232],{"class":164},"})\n",[14,234,235],{},"From a test, for grading:",[30,237,239],{"className":145,"code":238,"language":147,"meta":35,"style":35},"it(\"matches the reference implementation\", async ({ workspace }) => {\n    const { axon } = await Axon()\n    await axon.request(\"Implement parseLine in src\u002Fparser.ts\")\n\n    const expected = await Bun.file(\".\u002Ffixtures\u002Fexpected\u002Fparser.ts\").text()\n    const actual = await workspace.read(\"src\u002Fparser.ts\")\n\n    observe(\"exact_match\", actual.trim() === expected.trim())\n})\n",[17,240,241,272,295,316,322,351,377,382,415],{"__ignoreMap":35},[39,242,243,246,248,251,254,257,260,263,266,269],{"class":41,"line":42},[39,244,245],{"class":45},"it",[39,247,195],{"class":164},[39,249,250],{"class":55},"\"matches the reference implementation\"",[39,252,253],{"class":164},", ",[39,255,256],{"class":177},"async",[39,258,259],{"class":164}," ({ ",[39,261,262],{"class":201},"workspace",[39,264,265],{"class":164}," }) ",[39,267,268],{"class":177},"=>",[39,270,271],{"class":164}," {\n",[39,273,274,277,280,284,287,289,292],{"class":41,"line":49},[39,275,276],{"class":177},"    const",[39,278,279],{"class":164}," { ",[39,281,283],{"class":282},"s9McN","axon",[39,285,286],{"class":164}," } = ",[39,288,198],{"class":154},[39,290,291],{"class":45}," Axon",[39,293,294],{"class":164},"()\n",[39,296,297,300,303,305,308,310,313],{"class":41,"line":59},[39,298,299],{"class":154},"    await",[39,301,302],{"class":201}," axon",[39,304,205],{"class":164},[39,306,307],{"class":45},"request",[39,309,195],{"class":164},[39,311,312],{"class":55},"\"Implement parseLine in src\u002Fparser.ts\"",[39,314,315],{"class":164},")\n",[39,317,318],{"class":41,"line":68},[39,319,321],{"emptyLinePlaceholder":320},true,"\n",[39,323,324,326,329,332,334,336,338,340,342,345,347,349],{"class":41,"line":84},[39,325,276],{"class":177},[39,327,328],{"class":282}," expected",[39,330,331],{"class":164}," = ",[39,333,198],{"class":154},[39,335,202],{"class":201},[39,337,205],{"class":164},[39,339,208],{"class":45},[39,341,195],{"class":164},[39,343,344],{"class":55},"\".\u002Ffixtures\u002Fexpected\u002Fparser.ts\"",[39,346,216],{"class":164},[39,348,219],{"class":45},[39,350,294],{"class":164},[39,352,353,355,358,360,362,365,367,370,372,375],{"class":41,"line":95},[39,354,276],{"class":177},[39,356,357],{"class":282}," actual",[39,359,331],{"class":164},[39,361,198],{"class":154},[39,363,364],{"class":201}," workspace",[39,366,205],{"class":164},[39,368,369],{"class":45},"read",[39,371,195],{"class":164},[39,373,374],{"class":55},"\"src\u002Fparser.ts\"",[39,376,315],{"class":164},[39,378,380],{"class":41,"line":379},7,[39,381,321],{"emptyLinePlaceholder":320},[39,383,385,388,390,393,395,398,400,403,406,409,411,413],{"class":41,"line":384},8,[39,386,387],{"class":45},"    observe",[39,389,195],{"class":164},[39,391,392],{"class":55},"\"exact_match\"",[39,394,253],{"class":164},[39,396,397],{"class":201},"actual",[39,399,205],{"class":164},[39,401,402],{"class":45},"trim",[39,404,405],{"class":164},"() === ",[39,407,408],{"class":201},"expected",[39,410,205],{"class":164},[39,412,402],{"class":45},[39,414,222],{"class":164},[39,416,418],{"class":41,"line":417},9,[39,419,232],{"class":164},[14,421,422],{},"The test can read both sides. The agent can only read one.",[113,424,426],{"id":425},"judges","Judges",[14,428,429],{},"When a benchmark grades with a model rather than an assertion, the rubric belongs here:",[30,431,433],{"className":145,"code":432,"language":147,"meta":35,"style":35},"const rubric = await Bun.file(\".\u002Ffixtures\u002Frubric.md\").text()\n",[17,434,435],{"__ignoreMap":35},[39,436,437,440,443,445,447,449,451,453,455,458,460,462],{"class":41,"line":42},[39,438,439],{"class":177},"const",[39,441,442],{"class":282}," rubric",[39,444,331],{"class":164},[39,446,198],{"class":154},[39,448,202],{"class":201},[39,450,205],{"class":164},[39,452,208],{"class":45},[39,454,195],{"class":164},[39,456,457],{"class":55},"\".\u002Ffixtures\u002Frubric.md\"",[39,459,216],{"class":164},[39,461,219],{"class":45},[39,463,294],{"class":164},[14,465,466,467,470],{},"Two things matter about judges beyond where the rubric lives. The judge must be held\n",[123,468,469],{},"constant"," across the matrix — if the assessor varies alongside the subject, the measuring\ninstrument is changing with the thing being measured, and the numbers compare nothing. And\nits token spend is recorded separately from the subject's, so the cost of grading never\ncontaminates the cost of the work.",[113,472,474],{"id":473},"what-does-not-belong-here","What does not belong here",[14,476,477,478,205],{},"Anything the task genuinely requires. If the agent needs a config file, a dataset, or a\ndependency to do its job, that is part of its world and belongs in\n",[479,480,482],"a",{"href":481},"\u002Fdocs\u002Fv2\u002Fbench\u002Fworkspace",[17,483,484],{},"workspace\u002F",[14,486,487],{},"The test is simple: would a person doing this task by hand need it? If yes, it is the\nworkspace. If it only exists to check their work, it is a fixture.",[489,490,491],"style",{},"html pre.shiki code .sCudf, html code.shiki .sCudf{--shiki-default:#DCDCAA}html pre.shiki code .sKc5r, html code.shiki .sKc5r{--shiki-default:#CE9178}html pre.shiki code .sOLPB, html code.shiki .sOLPB{--shiki-default:#6A9955}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sYbnZ, html code.shiki .sYbnZ{--shiki-default:#C586C0}html pre.shiki code .sTNBD, html code.shiki .sTNBD{--shiki-default:#D4D4D4}html pre.shiki code .sNl3T, html code.shiki .sNl3T{--shiki-default:#4EC9B0}html pre.shiki code .scz_3, html code.shiki .scz_3{--shiki-default:#569CD6}html pre.shiki code .s3F5K, html code.shiki .s3F5K{--shiki-default:#9CDCFE}html pre.shiki code .s9McN, html code.shiki .s9McN{--shiki-default:#4FC1FF}",{"title":35,"searchDepth":49,"depth":49,"links":493},[494,495,496,497],{"id":115,"depth":49,"text":116},{"id":135,"depth":49,"text":136},{"id":425,"depth":49,"text":426},{"id":473,"depth":49,"text":474},"Author-side inputs — setup data, rubrics, expected outputs. The agent never sees them.","md",{},"\u002Fdocs\u002Fv2\u002Fbench\u002Ffixtures",{"title":5,"description":498},"docs\u002Fv2\u002Fbench\u002Ffixtures","BvwzpswDhyRcWrHwdzkRClMDYFpzJbPekkybw732Ns8",1785237008166]