{ "schemaVersion": 2, "experiment": "vanilla_vs_plugin", "generatedAt": "2026-10-04T09:31:47.571Z", "pairs": [ { "id": "signal_lantern--gpt-6-astra--r01", "task": "signal_lantern", "taskTitle": "Stylized signal lantern", "model": "gpt-6-astra", "modelTitle": "GPT 6 Astra", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "signal_lantern", "repetition": 1, "baselineAutomatedScore": 89.83, "candidateAutomatedScore": 100, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "plugin", "plugin" ], "criterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_glass_light", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_glass_light", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 7, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 6, "surfaceFinishAndShading": 7, "materialTextureQuality": 5, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 9 }, "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 9, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.97, "scores": { "A": { "taskFidelity": 7, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 6, "surfaceFinishAndShading": 7, "materialTextureQuality": 5, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 9 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 9, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "unclear", "B": "pass", "evidenceA": "Base, manufactured body, amber chamber, cap, and handle are present; an internal light cannot be distinctly identified.", "evidenceB": "All required components are visible, including a distinct flame and burner inside the chamber." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "The brass handle terminates at visible pivot fittings on both sides.", "evidenceB": "The dark arched handle connects to brass pivot fittings on both sides." }, { "criterionId": "lantern_glass_light", "A": "fail", "B": "pass", "evidenceA": "Dense amber speckling obscures the interior; highlights do not establish a distinct internal flame or light.", "evidenceB": "Transparent amber glass clearly reveals a warm flame above a modeled burner across the upright views." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded teal housings, layered base rims, brass guards, pivots, and controls form a finished stylized prop, despite the grainy chamber.", "evidenceB": "Smooth curved housings, clean rim details, contrasting metal fittings, and readable internal construction deliver a polished stylized finish." } ], "majorDefects": { "A": [ "The heavily speckled chamber reads as an opaque textured insert more than glass and conceals the required internal light." ], "B": [] }, "rationale": "B wins through clear warm glass, a distinct flame and burner, and cleaner material separation. A has strong manufactured detail and consistent construction, but its noisy chamber undermines the lantern’s central functional feature." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "plugin", "scoresByMode": { "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.5, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 6.5, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 6, "surfaceFinishAndShading": 7, "materialTextureQuality": 5, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 9 } }, "result": { "winner": "A", "confidence": 0.97, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.5, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "B": { "taskFidelity": 6.5, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 6, "surfaceFinishAndShading": 7, "materialTextureQuality": 5, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "unclear", "evidenceA": "Separate base, manufactured body, transparent chamber, flame and burner, cap, and handle are clearly visible.", "evidenceB": "Base, body, chamber, cap, and handle are visible; the densely speckled chamber prevents confirmation of a distinct internal light." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "The arched handle terminates at visible side pivots on both sides of the cap assembly.", "evidenceB": "Both brass handle arms connect to visible side pivot fittings." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "fail", "evidenceA": "Warm transparent glass reveals a distinct flame and layered burner across the elevation views.", "evidenceB": "The chamber appears densely granular and nearly opaque; highlights do not establish a distinct flame or internal light." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Smooth domed forms, rounded base edges, restrained brass trim, vents, pivots, and small fittings create a finished stylized prop.", "evidenceB": "Smooth stepped housings, rounded edges, vent details, handle grip, and control fittings show substantial refinement despite the chamber material problem." } ], "majorDefects": { "A": [], "B": [ "Dense granular chamber shading undermines polished glass readability and obscures the required internal flame or light." ] }, "rationale": "A wins through clear warm glass, an unmistakable flame, and cleaner material separation across views. B has convincing manufactured detailing and smooth bodywork, but its noisy, opaque-looking chamber compromises the lantern's central functional feature." } } ] }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "vanilla": { "score": 89.83, "rawScore": 89.83, "hardGate": false, "rawHardGate": false, "seconds": 451, "triangles": 22312, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "4e1930b385de222e88d297e87df1b5a118233037f85790d34897c8b675db5d63", "front": "f964cdcda1ab4634c1eb1c54e0e19b2760b207c30108db30c938450955eb29d7", "back": "b5aa18620bd7eea64175437a697ea93f32e6576094ce44e723f8a3d2cb4da155", "left": "b8a6050f590bd0c13224f9d7d0f4faad4be0268eff07beb74457d31f18314b2f", "right": "7053c0b77fd4ad59781294b906de95f9d4bf41fe387f48d7258583b4444f6b24", "top": "22ca2e2e02bd23ca83debd8d58240ead8bb7f1652375644db796f6dda08de259" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "b5eae5bb974cf2d98bc36ad95af89632b9ccbbc0e7bd2d854d1f42c7c25a4761", "front": "a2d9abbe178bef346af349690468bd0681573c0466c1413559de1801070dec11", "back": "743d38ce83a9f74c85186eb5bd8e6484cbc6767ff26da1ee1398659fdd86a50a", "left": "47fa306fea7fc297727a936253903f815cc7aabc65c0104ac21f41951ea86029", "right": "c7140e273bf1e95e90b4d2c16a90192bf57ef7ba74585744bf98a3d381329fb9", "top": "0bf38c6111eb38f85b56d7bddc9f421dac2d3811f130ead95aaf1a315e5f4854" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "824db82ea6c86efe0c400978c220281101e4829a3b4f96cf7cd25c04858c93b2", "front": "d53ffd16d9d58cda6c8f365ab23276c931069586a8d3966636a66e5615beb528", "back": "6e7f894b85a9d30048f1e8795b4f675e0df6871a0fa62e511d24c3cb10232b76", "left": "c21159c2db2d3d52d16cdb6a1f1fe5ea9bc42e57bda745484024021d8587fc5d", "right": "1d780289eae13d248a04421276b529100354a823a25c64a86f8dbcfeb3321782", "top": "caf3d20ade2441abc84de408c92c816cf6d299cf939d9786c29568052863554c" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "73f9f6c35ae2d62d347dc43a7ca00511d58ad1ef6feb0e997039bca4c9c3c3a2", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [ "Studio floor" ], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [ "Studio floor" ], "sourceSha256": "73f9f6c35ae2d62d347dc43a7ca00511d58ad1ef6feb0e997039bca4c9c3c3a2" }, "artifactHashes": { "create_asset.py": "0d119d1b864aded27debe21fb035e560205e7ab247bb7a2a1f9f087c867d45b4", "asset.blend": "73f9f6c35ae2d62d347dc43a7ca00511d58ad1ef6feb0e997039bca4c9c3c3a2", "asset.glb": "13a2abb4bbf156359bfddbcf76131aa32545697892925b11812b2af47c512bd9" }, "failedChecks": [ { "id": "semantic_part_coverage", "passed": false, "earned": 20.833333333333336, "possible": 25, "detail": "5/6 required semantic part groups found" }, { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "786 invalid vertices, degenerate faces, or zero-length edges" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "200.000m maximum extent; limit 3m" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "agentVersion": "codex-cli 0.160.0", "modelRequested": "gpt-6-astra", "modelsObserved": null, "rawScorerVersion": 5 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 648, "triangles": 23544, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "766da01849fa7eb287e5d15b41c3b20d0e1135fe62bd967097685c4a3a2aa9f2", "front": "8c8ced04c8642ec86631b8fba26a8b23bae4ff3c5de93bed95edb6ddf0875bc6", "back": "b30a8ab88c594bbebfce515c0133781bf6de4dae7d48d150d9fa1255292f92d7", "left": "f1922f7db8f54115289814eeddcffeca64ae98010ee7b3c5bd9a1dbd3234d22b", "right": "a543d3330e0ee475cd94507ae51eec2c24d9f1ed8f365ea9c29e966131426aaf", "top": "eb264386c135c4c1c3b3053b5a29c666302194f8143a3ec50c27c5764d1d8785" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "5027789b5b3e3f04c85281948b97cae89e07eee25a9d2c8bf4064cbd54d048cb", "front": "edf01334d47a3edece227fbcd3e66cb9e3ddc186a0e5073ac36394389a585657", "back": "3128ee0aad1bb19549d16a4e5a73c67a2ca25da5fb4b45a27aa2b1aa362da422", "left": "3845ac734e37765413f0df958bc4b20b365e6c8d748db6bbe834eaf4d2ed74c2", "right": "92f474f4d89b8f0235744bd12eeb0ae952fcc451e6d34589ac858a509dd58ffe", "top": "f5cd312457a819d9ce6c4f48b24a2a0990b268bf2bd820f04de6be92e15d0d1b" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-astra--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "77b17c86312caacb2741c87410bc4dd98d637fcbb008542e52f0ce26ae490a1d", "front": "ac8567266c3a82d9a3aa7f8a5a2301f0aa51ba3427e657c81b4b98a3e32cfa96", "back": "8b4bbe31c37aa4ecd31af100b987209c69284b4f4ceb817870f993f413b1302f", "left": "89094ee11dc6fc779322cc01eacf123ba2981ecb0925a1eba87ed6d0a5415ae9", "right": "c105aa10f44b0da8e9ec1530f786bcce072df7c011b77ede3c2356666e990c70", "top": "104cb410bfc68b5f51249f9e3194ddc0bfe805a3c62c64230ccf5469992be831" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "9d1d7fbb3b8ef16619fc3ba535917087544814c0654bfd8bb9ab985300522668", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "9d1d7fbb3b8ef16619fc3ba535917087544814c0654bfd8bb9ab985300522668" }, "artifactHashes": { "create_asset.py": "9af796ce267089e91619d7e1236cf2bd1cdeb874acb5b5be3ed268d5125d579d", "asset.blend": "9d1d7fbb3b8ef16619fc3ba535917087544814c0654bfd8bb9ab985300522668", "asset.glb": "b31f44e2fe927a8584af419b5e65987107c0dc7bea2cd4d4c140780727968afc" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "agentVersion": "codex-cli 0.160.0", "modelRequested": "gpt-6-astra", "modelsObserved": null, "rawScorerVersion": 5 }, "criteria": [ { "id": "lantern_parts", "label": "lantern parts", "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_handle_attachment", "label": "lantern handle attachment", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_glass_light", "label": "lantern glass light", "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_finish", "label": "lantern finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The plugin result wins through clear warm glass, a distinct flame and burner, and cleaner material separation. The no-plugin result has strong manufactured detail and consistent construction, but its noisy chamber undermines the lantern’s central functional feature." }, { "id": "signal_lantern--gpt-6.1-sol--r01", "task": "signal_lantern", "taskTitle": "Stylized signal lantern", "model": "gpt-6.1-sol", "modelTitle": "GPT 6.1 Sol", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "signal_lantern", "repetition": 1, "baselineAutomatedScore": 94, "candidateAutomatedScore": 95.83, "hardGates": { "baseline": false, "candidate": false }, "visualWinnerVotes": [ "plugin", "plugin" ], "criterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_glass_light", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_glass_light", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 9 }, "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.5, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.87, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 9 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.5, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Stepped base, separate body, enclosed chamber, flame, domed cap, and arched handle are visible.", "evidenceB": "Flared base, manufactured body, glass chamber, flame, vented protective cap, and arched handle are visible." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "Both handle ends terminate at visible side pivots beside the cap.", "evidenceB": "Both handle ends connect to projecting side brackets with visible pivot hardware." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Amber chamber surrounds a distinct flame, although dense grain weakens glass clarity.", "evidenceB": "Smooth amber glass has readable reflections and transparency, with a distinct flame and burner inside." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded enamel forms, metallic trim, cage rings, fasteners, and a small plaque establish a finished prop.", "evidenceB": "Smooth flared forms, refined rims, vent openings, rivets, brackets, and a contrasting handle grip establish a polished prop." } ], "majorDefects": { "A": [ "Conspicuous grain across the glass chamber undermines the polished finish and obscures internal construction." ], "B": [] }, "rationale": "B wins through clearer warm glass, smoother shading, and a stronger silhouette with a broad stable base and distinct vented cap. A has rich construction detail and consistent views, but its grainy chamber reduces material clarity and overall refinement." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "plugin", "scoresByMode": { "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.6, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 8.7, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8, "surfaceFinishAndShading": 7.8, "materialTextureQuality": 7.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.3, "multiviewConsistency": 9 } }, "result": { "winner": "A", "confidence": 0.78, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.6, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.7, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8, "surfaceFinishAndShading": 7.8, "materialTextureQuality": 7.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.3, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Flared base, separate body, amber chamber, flame, vented cap, and arched handle are visible.", "evidenceB": "Stepped base, manufactured body, amber chamber, flame, protective cap, and arched handle are visible." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "Both handle ends meet visible side brackets and pivot fittings.", "evidenceB": "Both handle ends connect to visible pivots beside the cap." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Warm translucent glass has smooth reflections and reveals a distinct flame and burner.", "evidenceB": "Amber glazing encloses a distinct flame, though dense granular shading weakens its glass appearance." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded enamel forms, smooth trim, vent openings, fasteners, and handle grip read as a finished stylized prop.", "evidenceB": "Smooth domed cap, rounded base, layered brass trim, fittings, and plaque provide finished secondary and tertiary detail." } ], "majorDefects": { "A": [], "B": [ "Pronounced granular chamber shading across multiple views reduces glass clarity and conflicts with the polished smooth finish." ] }, "rationale": "A wins through cleaner glass, smoother shading, and a substantial, balanced silhouette. B offers richer small construction details and consistent assembly, but its visibly grainy chamber makes the overall finish less polished." } } ] }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "vanilla": { "score": 94, "rawScore": 94, "hardGate": false, "rawHardGate": false, "seconds": 501, "triangles": 23330, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "f490f27df5774dc81f8c2ce8801c4cbeb909b29c60f984eabd5ec678ef57bd58", "front": "4503fa8c1e98ad264549c4b48cdce69efe6f7ba1ea7c6e9037156ee96d6a2669", "back": "843cb8deb044a0aec254baff6553b54463cc48ba0df6f3273d05c5a573b74be2", "left": "32c243f9167f51dd70ffb352ab293775b4de9c7b1725aaa24600008d21d48da0", "right": "07aab5c03bf7d45c7ca634177a0f838b6b3a7d389edfefac8411cf8202e5ca02", "top": "570be37d3d0314b1aa4643500a093b40a883d3ed915fff3e34de4c6baf0ee831" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "359d4f7ee50433125179437473fe9bff2d56e373e1a20d7187857de081ed5fb6", "front": "463225f7f5ae71caada5bca3ebf79cf9a9626ef0f87c322a4e409280b6292eb3", "back": "ec47477f3ba7e8f38c02ca602cecc63da10dd6a3eb7f3ce7a601758f8babe980", "left": "fa7b9d0c29dccc8b064bf6d172da344968108b3ab70170206c6ca4f26aef96b2", "right": "31ad523d961db7daf88229df79d30a4078de6d140603abf418823f2dfa929513", "top": "ea58779e1e764321a5155ff00ddbf142b38cc162bef714306e1e4ac9ab7965dd" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "9d15367659b68176043497015c067ad668f08313790b2b17dda43e27aa525664", "front": "fc4fa1cd4b6c2a4b1facb2618ebd390e9b29f339921f498b461087b43f68188f", "back": "d56710dab1f8ef22e484c52d21df46a93610de1ad123514d065dad0ac2fd6bc9", "left": "29ad31ad8c27ad06204b4b8c4d303192f587b102dcb0607352b2ed63d2f34360", "right": "2293845647d44dcffb8ad9ccc353c3fdfbec4e85baffbb973df8af0f8f8700fc", "top": "aa858f1d8e1bf3220bff60b7c7524ca10b372a8c45d9820ef6c71a357e401852" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "6fcc246b1f4095c88b625cfee9b852deb64c44fcb3cf2f71612b712c911e81e7", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [ "Studio | ground" ], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [ "Studio | ground" ], "sourceSha256": "6fcc246b1f4095c88b625cfee9b852deb64c44fcb3cf2f71612b712c911e81e7" }, "artifactHashes": { "create_asset.py": "afeb9018c74cf436b36622db9f15abc0f8d294f497042fab68dca451d813b021", "asset.blend": "6fcc246b1f4095c88b625cfee9b852deb64c44fcb3cf2f71612b712c911e81e7", "asset.glb": "883f99342966ffdc2d66046f8ae6d2ff3c8f83ad40a5335b788d5e52286c1767" }, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "230 invalid vertices, degenerate faces, or zero-length edges" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "200.000m maximum extent; limit 3m" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "agentVersion": "codex-cli 0.160.0", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 5 }, "plugin": { "score": 95.83, "rawScore": 95.83, "hardGate": false, "rawHardGate": false, "seconds": 692, "triangles": 22532, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "aa4892874533205799efca0ef46e147415f384c4c0e5392d872d7c4a6a079207", "front": "404ee4e2b9d1ea8ae633b50b4a2494a48d6b1da51263f840bd26953bcb1a0421", "back": "f45b279a73e9ab86600bafd576e0299beab48ea53fc23c15c6c735533e3c005a", "left": "d67872605688864e0501f8b8b6acf622a6352fd18bbdf99a86f8ed61bfa61ae9", "right": "58ccff8e846ffb3e25517729f3c7284a96e14d2132cc8b15e6c47bb17651eacc", "top": "171d1ab8e208d336d5b21cda5a8ce1a48b4eddcbec268f5f4879f43373d9035e" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "96a278e6ddad8ce8060dbb5902444a836b98598ed362cd649ce449f78ee804ca", "front": "b56ba646f8d54f41e9e7f4308bc7bb7e5924abb6fefcf8ffbbd72eaffd41da8e", "back": "b3454645b4b2437a54ea65a9c86a1e0e9f658255f1d47d8348185872d0ca7d47", "left": "d1f36738ecd672dc047feb058848e869cadc90cf09e01ec5d03d58d251367718", "right": "d2807556b96477139b0e18b599e812e44ea1b16000e899748f2b3f909e91e182", "top": "9b970d8244d2a823ed3e8e777fa62006abe5fc2fe6a60877e3bb60304cf028de" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6.1-sol--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "2ea1e0f40a34aed1adaac2dc9f90e33bfc947ccd1ebffc4666d20a6a65984b8e", "front": "bda7f6620b1fe8ba2356ea6dd8d3881cb48d4d6bd3742e7e4592e9f37de5ad00", "back": "bb61ed7e46fa1ec3495595b28805bf5548ee8fa1200ee318cf42370e3c9266b5", "left": "3cdcba59b9433ae9e9a5d8653964578871147c344b9872a302d15b8821f7efaf", "right": "96586f5e0d1876d75637c5227ea53e52c18a03ce6d32b3dae8c31bd140ccf024", "top": "75a53c99e9d071767c2d1cf508b6f63e5269dabee17d14805a591e02b1a582cd" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "b0a6c4497d91897167a0e3fc76b6fee65fd406fb4f92b1a3d38f89634e01c3d5", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "b0a6c4497d91897167a0e3fc76b6fee65fd406fb4f92b1a3d38f89634e01c3d5" }, "artifactHashes": { "create_asset.py": "bc1d651cb28cd540605a1dafd5dfffba48a0de00aea62f41b67eed0510410731", "asset.blend": "b0a6c4497d91897167a0e3fc76b6fee65fd406fb4f92b1a3d38f89634e01c3d5", "asset.glb": "d44fe732e9d58419ff4fba1f1f6173edbd629f1a5bed6ff076968b43123d35fc" }, "failedChecks": [ { "id": "semantic_part_coverage", "passed": false, "earned": 20.833333333333336, "possible": 25, "detail": "5/6 required semantic part groups found" } ], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "agentVersion": "codex-cli 0.160.0", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 5 }, "criteria": [ { "id": "lantern_parts", "label": "lantern parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_handle_attachment", "label": "lantern handle attachment", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_glass_light", "label": "lantern glass light", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_finish", "label": "lantern finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The plugin result wins through clearer warm glass, smoother shading, and a stronger silhouette with a broad stable base and distinct vented cap. The no-plugin result has rich construction detail and consistent views, but its grainy chamber reduces material clarity and overall refinement." }, { "id": "signal_lantern--claude-opus-5-5--r01", "task": "signal_lantern", "taskTitle": "Stylized signal lantern", "model": "claude-opus-5-5", "modelTitle": "Opus 5.5", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "signal_lantern", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "vanilla", "vanilla" ], "criterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_glass_light", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_glass_light", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.6, "materialTextureQuality": 7.8, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "plugin": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.3, "craftsmanshipAndDetail": 8.2, "materialsAndReadability": 8.2, "surfaceFinishAndShading": 8.3, "materialTextureQuality": 7.6, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.4, "multiviewConsistency": 9 } }, "result": { "winner": "A", "confidence": 0.78, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.6, "materialTextureQuality": 7.8, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.3, "craftsmanshipAndDetail": 8.2, "materialsAndReadability": 8.2, "surfaceFinishAndShading": 8.3, "materialTextureQuality": 7.6, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.4, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Stepped base, red manufactured body, cylindrical glass chamber, internal light, protective roof, and arched handle are visible.", "evidenceB": "Stepped base, red body, rounded glass chamber, flame, protective roof, and arched handle are visible." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "Both handle ends meet visible brass side brackets at the roof rim.", "evidenceB": "Both handle ends connect to visible side pivot fittings near the roof edge." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Warm translucent chamber surrounds a distinct bright internal light and visible lower burner.", "evidenceB": "Amber glass surrounds a clearly visible bright teardrop flame." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Smooth roof and base transitions, layered brass trim, fasteners, vents, and controls establish a finished prop.", "evidenceB": "Smooth curved globe, rounded roof, beveled base, cage wires, rivets, and controls establish a finished prop." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "A wins narrowly through richer manufactured layering, more substantial protective framing, and clearer brass, enamel, and dark metal separation. B has an appealing rounded globe and clearer flame shape, but its pale roof finish and repeated small fittings feel less refined. Both remain coherent across views; grainy glass slightly weakens both finishes." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.2, "constructionPlausibility": 8.1, "craftsmanshipAndDetail": 8.2, "materialsAndReadability": 7.6, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 7.3, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.3, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 7.8, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.8, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.2, "constructionPlausibility": 8.1, "craftsmanshipAndDetail": 8.2, "materialsAndReadability": 7.6, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 7.3, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.3, "multiviewConsistency": 9 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 7.8, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Stepped base, red housing, warm barrel-shaped chamber, visible flame, cap, and arched handle are present.", "evidenceB": "Broad stepped base, red housing, warm cylindrical chamber, internal light, layered cap, and arched handle are present." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "Both handle ends visibly meet side pivot fittings on the cap.", "evidenceB": "Both handle ends connect to distinct brass side brackets." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Amber chamber contains a distinct bright flame, although grainy shading reduces glass clarity.", "evidenceB": "Warm translucent chamber reveals a distinct bright internal light; surface grain slightly weakens the glass finish." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded chamber, smooth cap, rolled base edges, cage wires, and small fittings produce a finished stylized prop.", "evidenceB": "Smooth flared cap, rounded base tiers, brass edging, vent openings, brackets, and fasteners provide polished manufactured detail." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins through clearer manufactured joints, stronger material separation, and more purposeful cap and signal details. A is complete and consistent, but its grainier chamber and less differentiated fittings reduce refinement. Both satisfy all visible requirements." } } ] }, "votes": { "baseline": 2, "candidate": 0, "tie": 0 }, "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 153, "triangles": 19380, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "6d8135f3a30916c0a6b23c76c0f35a89e9c3950268ce4967009447f2496d8f48", "front": "c27afa81befb02c80055fe376dc8d37be1485a104845962ee22f5c22806b24cf", "back": "c377903cc162efeb393e0ac1f8060c392a80bde4c2f9781bf8049516d20830fd", "left": "aeb4304cfdbdd45de9c2dfe10a09981b6ee1e2afdd246def22bc6e4419485cc1", "right": "ac54bc4b14dae39dfb63f057fc3a09362088cd1f2e875d0f16c4d3abefd7239d", "top": "e711b58d277124c642dbc037345a888cd329c19c4a438b34c2a6a95db23df67e" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "c4c7291472d6585a2c84154611d94649e41259ae7dc77e1e4904531b35c9aaec", "front": "6d9a9326b56a700d922ebb35aad4741e892e5b95417d804c969ac23d42b01579", "back": "88d442630014d1c21ef51568077cb721d2a0fc642865267644b532ce4daa5f86", "left": "1f2668ab2d02d01350a400fe3988682d9618f39cf7290563e1e65d30b7fabfd3", "right": "efe97cbf5531c2f420c3c484341f6ffdd5f031c468c921a699769e6300dd9307", "top": "1f85c4404119bad96defbd6ce18f64f3d5f6cf90d29a3e92788aac741f4342d5" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "e3b06ecea3857f918db77c938e942325b69457cbd52ffcebb9f8965ff920e1c0", "front": "4a142bfc1176a4a23192fc836165d06332029f26eda67d6365eb7f5b5b502963", "back": "b7625d663a3ee480260ab933cd3aab6b719939ab37b296963466840d050f32bf", "left": "c8dcdb7a4ea8af64eba997082f4519e92a5a9ed3b44b3c2848b4d5a8b9b4c9ec", "right": "ccda885a7207f1020d8153a018abe8ea2e1fb12f978373b639e141f39d81661b", "top": "534041c26a50bf18a7c5d7a73f69d3c4fbfd173f717bf96541e7ab8dba401185" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "8832751d8e73805743e4918507f128127afca35d14194cf0aad50a510df24b23", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "8832751d8e73805743e4918507f128127afca35d14194cf0aad50a510df24b23" }, "artifactHashes": { "create_asset.py": "7228157ad24c7b99fed2e1e94c35b5f5b61c1aa2b8e4e696e92ad66d470f182a", "asset.blend": "8832751d8e73805743e4918507f128127afca35d14194cf0aad50a510df24b23", "asset.glb": "e7b2c353aa9b23c1a2fd7cdce4f671542e7d41657a85690c5b3fb6c8e91311fb" }, "failedChecks": [], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 5 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 172, "triangles": 21124, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "d4a35040ce01d1c453f1c2704b19e3173f4707eb19dfc9a0a2983d30b5e6c67f", "front": "8aab6b57d5fa4f76a09e0b26039205bca1d1c9525d11e92a6527132046170281", "back": "4699b144b41cd64c85f3ceb40bd1acd541c709640b57c4b2839282c5bd959b9a", "left": "b3c20e285e9b9c311aff76cffd115fbfb9ac2bb4155a348e8f24ea3c80c09f03", "right": "ee681ebb69cd66b42ca2ab6bf51e220854dffadef7fb92880ab1f4a6bcb88713", "top": "f1603dd0f09b295f4381881cdec776000993cc2089f649c7a1ed6c6963e28e21" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "5e0ff655327864a75970bace3c62158eeb80c8621b09df86890309b8309e64fa", "front": "2e2d670daf3a4d86703ba901a06dfa0c3bc8f592f36af3c11e635ba2c425999c", "back": "5e272918ac21dc77df4b10f540b71c9f596f471d6505d5cd727c5a9814d1baa0", "left": "b6ce462022684fb4950d9f9574aa61cd1a433aab8e4879af3de5739fbba1c81e", "right": "6b9dc61bd7792bd2e3f6597b366d1777ac1c34efb6885946702c979328fc9be8", "top": "b939c306d3e2623f9df43bfb20447618fbdc99ed47a225254d6cab3e22ddd9fe" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-opus-5-5--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "3f5672d719e6f941f4c6de8f09638a52e45a498f48fb35ac613a6160ea989150", "front": "812cbb945ec2896339edd8dba14883ed010b408c32a8f01b1c2a5c58fe5ddd03", "back": "25005b776904f611716a0b98ba9492a3c655b333cd59e303d311dfb50449c23e", "left": "e1f847ff558d70923560879ef0bb5f1af413f0f2396f988f6e1ee4eea9fb5d57", "right": "bba0b2be7a0bf778eaf8932307be9cd20c1f0a9b75b82b130a039aef2a4982a3", "top": "0673b56bbe75f34a5d8026597ee364ff4b81704d80cf054dd904ca7ca564e7a2" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "da28bab2f29c0c5ea3a116e56939116daca1873efe9976786bbe862573249857", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "da28bab2f29c0c5ea3a116e56939116daca1873efe9976786bbe862573249857" }, "artifactHashes": { "create_asset.py": "82266f68ec09e00632bd36f9be375f3539947db1229ca2e766149f6a37449adb", "asset.blend": "da28bab2f29c0c5ea3a116e56939116daca1873efe9976786bbe862573249857", "asset.glb": "39bd6434a1db26119532a79e1fb84e3cee67498ccc39ebbcca00108be1caa920" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 5 }, "criteria": [ { "id": "lantern_parts", "label": "lantern parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_handle_attachment", "label": "lantern handle attachment", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_glass_light", "label": "lantern glass light", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_finish", "label": "lantern finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The no-plugin result wins narrowly through richer manufactured layering, more substantial protective framing, and clearer brass, enamel, and dark metal separation. The plugin result has an appealing rounded globe and clearer flame shape, but its pale roof finish and repeated small fittings feel less refined. Both remain coherent across views; grainy glass slightly weakens both finishes." }, { "id": "signal_lantern--claude-sonnet-5-5--r01", "task": "signal_lantern", "taskTitle": "Stylized signal lantern", "model": "claude-sonnet-5-5", "modelTitle": "Sonnet 5.5", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "signal_lantern", "repetition": 1, "baselineAutomatedScore": 96.12, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "plugin", "plugin" ], "criterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_glass_light", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "lantern_glass_light", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "plugin", "scoresByMode": { "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.7, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.3, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 7.2, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8, "multiviewConsistency": 8.8 } }, "result": { "winner": "A", "confidence": 0.87, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.7, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.3, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 7.2, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8, "multiviewConsistency": 8.8 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Distinct stepped base, framed glass body, bright internal flame, red protective cap, and arched handle.", "evidenceB": "All required components are visible, including a tall guarded glass chamber and internal flame." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "The arch terminates at visible side pivots beside the cap; side views show the mounting hardware.", "evidenceB": "Both handle ends meet visible brass side brackets, confirmed by the side views." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Amber translucent chamber surrounds a clearly distinguished cream-colored luminous flame.", "evidenceB": "Warm transparent glass reveals a distinct flame and small burner beneath it." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded enamel cap, beveled stepped base, trim rings, fasteners, and handle grip produce a finished stylized prop.", "evidenceB": "Smooth enamel and metal surfaces, cage rings, attachment hardware, and base fasteners provide visible finish and detail." } ], "majorDefects": { "A": [ "The large flame has an angular outline, and the broad amber chamber reads somewhat flat in elevation views." ], "B": [ "The brass crown above the red cap appears separated by a dark gap, without a clearly visible supporting connection.", "The slender body and oversized handle leave the upper silhouette less balanced and substantial." ] }, "rationale": "A wins through stronger proportions, more developed base and cap construction, and clearer integration of its components. B offers useful protective cage detail and readable glass, but its apparently floating crown and less balanced silhouette weaken construction plausibility and overall finish." } }, { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 8.7, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 8.3, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.1, "multiviewConsistency": 9 }, "plugin": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 8.1, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.6, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.76, "scores": { "A": { "taskFidelity": 8.7, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 8.3, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.1, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 8.1, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.6, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Separate foot, red body collars, warm chamber, internal flame, domed cap, and arched handle are visible.", "evidenceB": "Layered base, manufactured collars, glass chamber, internal light, broad cap, and handle are clearly present." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "Both handle ends terminate at visible side pivots beside the cap.", "evidenceB": "The arch connects to paired side pivots, with mounting brackets readable in side views." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Amber translucent glass surrounds a distinct cream-colored flame and visible burner.", "evidenceB": "Warm translucent glazing encloses a distinct bright flame-shaped light above a burner." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Smooth rounded cap and base, metallic trim, protective hoops, and small fittings give a finished prop appearance.", "evidenceB": "Rounded manufactured layers, cap fasteners, handle brackets, and base hardware read as polished construction, although the flame outline is angular." } ], "majorDefects": { "A": [], "B": [ "The large flame has a conspicuously polygonal outline that weakens the polished smooth finish." ] }, "rationale": "B wins narrowly through a stronger cap-to-body silhouette, more substantial handle mounts, and richer manufactured detail around the base and crown. A has a smoother flame and useful protective hoops, but its thinner fittings and simpler construction feel less resolved. Both remain coherent across all six views." } } ] }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "vanilla": { "score": 96.12, "rawScore": 96.12, "hardGate": true, "rawHardGate": true, "seconds": 83, "triangles": 22700, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "151d46a65600b003fb33b94ca68187c0b2eec35717cb0081d6c57799b7f46002", "front": "a51d31d37cc49a5c28c5fbfde841893a50c4c98d78be3a994a8f9c90c22a2e22", "back": "e9a1fed10558276f9842ab07a9c4e949877acc96f11e239de01b85c6a514bc14", "left": "326c4b08adf2c39a0b8b55cb0ded9e7721429e8ab9692836fe97c9a4783c681f", "right": "470b213bf62c91e49bec05ed5f243d3895d0004ba46a5d819c851b2ce7ecee84", "top": "1ed703e3ebfd4fb30e42f4f6f692f7a01cdcf42a293ea95746e765001f24061a" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "154bc27699108e08b6e195f5c0bc3efac4e9872486af5f6a882c73d6e1cbc7b0", "front": "5e01a653d2b4760bbeab6ee6b30e27d8ce0545a8bf70b3b776e05b9bf5fce75b", "back": "6625e5b7b80c4cc07d7a1330dcffe26ef07dbe03d33e99640dfa1bca8ec6ab40", "left": "594d6896797a60b9f7d679255789523413fc0977ea970e439da1f63aa76b121c", "right": "789695a308e135be3a44eea53872c2b336741e9bdd545898f926a9f133a98c8e", "top": "41dee52ca0461c0d8d5aa23e7de17282ab284794ef3e104681889e9ffc3e14e1" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "203da6f37ea4cbb839011b8ec31fe63f0441be9f3967a93a735adf5fbf912d32", "front": "bf481cd8831eb28b1ca8c81254eac65a5a03e19f11d49cc76d55e916304839e5", "back": "27419bad7fe51d95147f11a62dedbc5aebf6efc2250d13aca01fc974f6e6e4a8", "left": "ba6961ce6c017c3814e8725a7c23c0f8bbc650fc3859b0d272255ca2cc0c1fc1", "right": "8ea5da2d160edbaa9da5c335e54d026105fd1145af0c87eefb53af8c06c6cc9f", "top": "63b5cd515f68de0455067f880ae52fc359ce1b59fd287c2e48a887b84046039f" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "bc1652f4131474ab4f776628e2988db6d6242e80f913965989fa56e05c5dc54f", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "bc1652f4131474ab4f776628e2988db6d6242e80f913965989fa56e05c5dc54f" }, "artifactHashes": { "create_asset.py": "3cca0b10a05fadb4981a163fac6310a366359df405091e5e05efd7e50e69bf70", "asset.blend": "bc1652f4131474ab4f776628e2988db6d6242e80f913965989fa56e05c5dc54f", "asset.glb": "bd846c7eed8291df1539c272e5989ce0fda776a77b33cdd19c4a897830cc6fb0" }, "failedChecks": [ { "id": "uv_coverage", "passed": false, "earned": 0.11695906432748537, "possible": 4, "detail": "1/57 meshes have UV layers; minimum ratio 0.60" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 5 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 106, "triangles": 17564, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "e82a60a3e2fe39af2ad4c8ac4aeda42b7856a3fdb137a7ca162560abec6b8ea1", "front": "d92a9bd4430cb06254f40663bb85516a2c7ce197f481200de58bd6a1c26c54f9", "back": "622f344b793911a713ed4a86aea38fe90f308101612b184833bafa871f0b4967", "left": "7ddfe28517e4e331e46a3af61facf8a096db8b2571c2b3484092a4c3f3869df0", "right": "8a0f264f265d73fb299b33ef1fffb7eb8f6975d827b020c1a398f18783214185", "top": "05b3d407424c8b60bcf2187401df9544c94bcb616332b51a78115246c713d8d2" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "b04e41df43d20de1b34cf41512b5f6ad71f477dc5d993bd7f5557d7b0b472bba", "front": "75a52ec928bf729e1268f88aafe8be961c4e88aa88f832d92c2c3fe3870196e2", "back": "33f831d53506dde684bf5bece48371cdaa9fffd063e386852f44a3697b9cc4c7", "left": "30b66066a957415d4bf509baa9fb6257eb240fa8addec1508072ebca6bdbfa1d", "right": "71652a66de4bc91784e913ff24a3fbf94586c36abb5d53b5c8dd2f6dc84b330c", "top": "4b8128ac17647e252cd935449c8e9fb3c7e6803573ebf12467cff7e69675fb46" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--claude-sonnet-5-5--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "fb1bfe896c7dff4b24232bd90714c2c66da329326ec226fd8ab65bbc753616f9", "front": "a29e04cb24dc7c12da521af1b63ac0f0d039d4f4d2f9ba00d724ec7a50e8186e", "back": "9a4503dc165d1e1ebfe21f532ee526d96062c6fff684057e55f60907a766ce6e", "left": "00254276f97ec1dbc1e04f2a96b25f3b71a063d538e5babf5896fede9c84fdf4", "right": "a86af1adceec860db4edc72c4867293882e2a051b775967706ac16db9af4549d", "top": "d0536d30c66b8a269bf2d4b7b819837be929f8f687a0c6f73230905fa157b60e" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "5228493fcc814c09c200a2961ce9a44f599e613ccac6c1b3faae2bac20f7b40e", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "5228493fcc814c09c200a2961ce9a44f599e613ccac6c1b3faae2bac20f7b40e" }, "artifactHashes": { "create_asset.py": "b1b6c561469367490125f4ebb3676f883b07d656d208000f2a14ddd305ec2d8c", "asset.blend": "5228493fcc814c09c200a2961ce9a44f599e613ccac6c1b3faae2bac20f7b40e", "asset.glb": "66528883a10b9bf1fce51973bd0367610f899b46c3b66a00cd7d437d2998266f" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 5 }, "criteria": [ { "id": "lantern_parts", "label": "lantern parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_handle_attachment", "label": "lantern handle attachment", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_glass_light", "label": "lantern glass light", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "lantern_finish", "label": "lantern finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The plugin result wins through stronger proportions, more developed base and cap construction, and clearer integration of its components. The no-plugin result offers useful protective cage detail and readable glass, but its apparently floating crown and less balanced silhouette weaken construction plausibility and overall finish." }, { "id": "signal_lantern--gpt-6-sol--r03", "task": "signal_lantern", "taskTitle": "Stylized signal lantern", "model": "gpt-6-sol", "modelTitle": "GPT 6 Sol (historical)", "repetition": 3, "cohort": "September 2026 quality study", "judgeCount": 3, "limitation": "Historical skill-workflow comparison, with the original model and evaluator. Three reviews of each pair. Scores are kept separate from the October pilot. Some judged views hide oversized studio floors; original delivery failures are retained. Run 3 selected by the gallery owner after viewing all three attempts. This is a curated example, not a random sample or an aggregate result. Earlier attempts remain in the archived gallery release.", "votes": { "baseline": 3, "candidate": 0, "tie": 0 }, "rawReview": { "taskId": "signal_lantern", "repetition": 2, "baselineAutomatedScore": 96, "candidateAutomatedScore": 100, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "baseline", "baseline", "baseline" ], "criterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "lantern_glass_light", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "lantern_finish", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 } ], "criticalCriterionMajorities": [ { "criterionId": "lantern_parts", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "lantern_handle_attachment", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "lantern_glass_light", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "skills", "B": "baseline" }, "decodedWinner": "baseline", "scoresByMode": { "skills": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 8.2, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8, "multiviewConsistency": 9 }, "baseline": { "taskFidelity": 9.2, "silhouetteAndProportion": 9.1, "constructionPlausibility": 9, "craftsmanshipAndDetail": 9.1, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 9.2, "materialTextureQuality": 8.7, "lightingAndPresentation": 8.6, "finalStageCompleteness": 9.2, "multiviewConsistency": 9.1 } }, "result": { "winner": "B", "confidence": 0.94, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 8.2, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8, "multiviewConsistency": 9 }, "B": { "taskFidelity": 9.2, "silhouetteAndProportion": 9.1, "constructionPlausibility": 9, "craftsmanshipAndDetail": 9.1, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 9.2, "materialTextureQuality": 8.7, "lightingAndPresentation": 8.6, "finalStageCompleteness": 9.2, "multiviewConsistency": 9.1 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Stable flared base, distinct body, amber chamber, flame, layered cap, and arched handle are visible.", "evidenceB": "Stepped base, detailed manufactured body, amber chamber, internal light, protective cap, and handle are visible." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "Both handle ends visibly meet circular side pivots at the cap rim.", "evidenceB": "Both handle ends connect through substantial side brackets and pivot hardware." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Warm amber glazing surrounds a distinct orange flame and visible burner.", "evidenceB": "Warm amber glazing surrounds a distinct pale luminous element and burner pedestal." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Smooth enamel surfaces, rounded trim, cap fasteners, and controlled highlights establish a finished stylized prop.", "evidenceB": "Rounded pressed-metal forms, clean rolled edges, vent details, fasteners, and articulated fittings establish a refined manufactured prop." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins through richer manufactured construction, stronger rounded cap shaping, substantial handle fittings, and coherent secondary and tertiary detail across views. A satisfies every requirement but its simple flared body and conical cap feel less developed. A's flame is more immediately readable; B has the stronger overall finish." } }, { "mapping": { "A": "baseline", "B": "skills" }, "decodedWinner": "baseline", "scoresByMode": { "baseline": { "taskFidelity": 9.1, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.8, "craftsmanshipAndDetail": 9, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.6, "lightingAndPresentation": 8.7, "finalStageCompleteness": 9, "multiviewConsistency": 9.1 }, "skills": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.4, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.3, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.4, "multiviewConsistency": 9 } }, "result": { "winner": "A", "confidence": 0.83, "scores": { "A": { "taskFidelity": 9.1, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.8, "craftsmanshipAndDetail": 9, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.6, "lightingAndPresentation": 8.7, "finalStageCompleteness": 9, "multiviewConsistency": 9.1 }, "B": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.4, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.3, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.4, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Distinct stepped base, detailed cylindrical body, amber chamber, internal light, rounded cap, and arched handle are visible.", "evidenceB": "Flared base and body, amber chamber, flame and burner, tiered cap, and arched handle are visible." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "Both handle ends terminate in visible side brackets with metallic attachment hardware.", "evidenceB": "Both handle ends connect to round side pivots beside the cap." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Warm reflective glazing encloses a visible pale light and burner; the central guard partially obscures the light in elevation views.", "evidenceB": "Amber glazing shows reflective highlights and a distinct orange flame, especially clearly in the perspective view." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded cap transitions, beveled base rings, layered fittings, vents, and small hardware form a polished finished prop.", "evidenceB": "Smooth curved surfaces, metallic trim, rivets, pivots, and burner detailing create a finished stylized prop." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "A wins through richer manufactured construction: layered body seams, vent details, substantial handle brackets, and more refined rounded transitions. B has a clear silhouette and readable flame, but its broad plain base and simpler cap construction provide less secondary form development. Both satisfy every visible requirement and remain coherent across views." } }, { "mapping": { "A": "skills", "B": "baseline" }, "decodedWinner": "baseline", "scoresByMode": { "skills": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.1, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7.7, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.6, "materialTextureQuality": 8, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.1, "multiviewConsistency": 9 }, "baseline": { "taskFidelity": 9.2, "silhouetteAndProportion": 9, "constructionPlausibility": 9, "craftsmanshipAndDetail": 9.1, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 9.2, "materialTextureQuality": 8.6, "lightingAndPresentation": 8.6, "finalStageCompleteness": 9.2, "multiviewConsistency": 9.1 } }, "result": { "winner": "B", "confidence": 0.93, "scores": { "A": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.1, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7.7, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.6, "materialTextureQuality": 8, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.1, "multiviewConsistency": 9 }, "B": { "taskFidelity": 9.2, "silhouetteAndProportion": 9, "constructionPlausibility": 9, "craftsmanshipAndDetail": 9.1, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 9.2, "materialTextureQuality": 8.6, "lightingAndPresentation": 8.6, "finalStageCompleteness": 9.2, "multiviewConsistency": 9.1 } }, "criterionResults": [ { "criterionId": "lantern_parts", "A": "pass", "B": "pass", "evidenceA": "Stable flared base, separate housing, amber chamber, flame and burner, layered cap, and arched handle are visible.", "evidenceB": "Foot, detailed cylindrical body, amber chamber, internal light and burner, shaped cap, and arched handle are visible." }, { "criterionId": "lantern_handle_attachment", "A": "pass", "B": "pass", "evidenceA": "Handle ends connect to paired side pivots beside the cap.", "evidenceB": "Handle connects through paired side brackets with visible pivot hardware." }, { "criterionId": "lantern_glass_light", "A": "pass", "B": "pass", "evidenceA": "Warm transparent chamber reveals a distinct orange flame above the burner.", "evidenceB": "Amber glass surrounds a distinct pale warm light above the burner." }, { "criterionId": "lantern_finish", "A": "pass", "B": "pass", "evidenceA": "Smooth cap and base, rounded trim, rivets, and clean material separation establish a finished stylized prop.", "evidenceB": "Rounded manufactured forms, rolled edges, vent details, fasteners, brackets, and layered materials provide polished refinement." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins through its more developed manufactured body, stronger rounded silhouette, and coherent construction details across views. A meets every requirement and has a clearer flame, but its plain flared housing and simpler cap provide less secondary and tertiary refinement." } } ] }, "vanilla": { "score": 96, "rawScore": 96, "hardGate": false, "rawHardGate": false, "seconds": 336, "triangles": 16404, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/hd/top.png" }, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "580 invalid vertices, degenerate faces, or zero-length edges" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "agentVersion": "codex-cli 0.155.0-alpha.9.2", "evaluatorFingerprint": "07a858a97099d561a6161b825aaac0de8fc8e1cc84a73194cbb121bb97cee695", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "rawScorerVersion": 5, "modelRequested": "gpt-6-sol", "sourceHashesRecordedAtGalleryBuild": true, "artifactHashes": { "asset.blend": "47d9178b88fe6ee4a1f455ecf3147992f3ae08681db72d060c7f34f93df8b079", "asset.glb": "19237347c28b24743b24643f4d1270b7ca205654a8f19e1bd9f58bbe95b58a84" }, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "47d9178b88fe6ee4a1f455ecf3147992f3ae08681db72d060c7f34f93df8b079", "sourceModified": false }, "imageHashes": { "perspective": "47e0366984dfbaf84b5a40d4be7070ad9edfaeea459a44ab78215cc4b99de385", "front": "ee6a526fb0a69c287d8b4edbc63a072eb17944765b51bc2f29e739dd6d978e9c", "back": "9c1b060e39cfc2c5bbf47393e0f691dd7c4d9f20d74b0ff992cf9d57d9ffdab3", "left": "c5d1977c8312d1baa7e755cffc7c3c749bb1c3009a8f74d6c13048d3772b528b", "right": "0915616b96f0fe8f9b89ae635f435212910f55604117ebe45a5c0d774e3afe3a", "top": "ae6812bbaa453ff37addd2385fcd920b0e5cd405e745d648a940372402b2e000" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "93b2de699c2f55f2235a29ae5b5bba5090b5e65e4df384cc29c342c74ac622b9", "front": "563b815fe2064211556fb768ea69c4baa9110e64af941067a92b791f82a9faa9", "back": "cb31fca25b19f1448264ccc7d8b804a482780aef19d5dd1153cb0e54c84629e1", "left": "24e2904c11878dd9f6a8d9fffff0fd9a7512518377366edc4d9961565d8672a4", "right": "81ba3c5b1c10cb6c02ac3464e3634c8c358014a5d63f7bd8d241da4682857c35", "top": "1c66d9e5b611bd89490df35b39d863aefb44284559ad30332cb227e93c467aa8" } }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 396, "triangles": 23200, "images": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/hd/top.png" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fd73795a211744fff6462d9ed0b38296bb8b9c01e44ad7ca5a6f3a232b7deec7", "agentVersion": "codex-cli 0.155.0-alpha.9.2", "evaluatorFingerprint": "07a858a97099d561a6161b825aaac0de8fc8e1cc84a73194cbb121bb97cee695", "generationTaskFingerprint": "dce1d6051aae46a8fe91eda478d2cc7e452e38c5cff9b0b342b2ef88efce6ca0", "rawScorerVersion": 5, "modelRequested": "gpt-6-sol", "sourceHashesRecordedAtGalleryBuild": true, "artifactHashes": { "asset.blend": "785b1a6297547841e516b417f022a5e0bfbeb83768e2c146c26cb6e913fc2f42", "asset.glb": "8bd1b3287c1097726d9a889ee1b940378f8852f5526aa2ae1c4e51c099ffdc62" }, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "785b1a6297547841e516b417f022a5e0bfbeb83768e2c146c26cb6e913fc2f42", "sourceModified": false }, "imageHashes": { "perspective": "938e0a1a247ef19346e7747058f86acecfeab07fa7556caab83cef5fe94b80f2", "front": "d059cd63cd38dbba455fe6a9e18bba652c0880fa2c417d51d243741a8bfb04ba", "back": "5128bd283ed159ba7339b4b980a04c5657819b0e2333931a508c67bfb55a90a5", "left": "4b40535d70a65ef1e04c5c3bab736775b0b075da97895c92300603382d9bf136", "right": "5a0b7f183ba333b46f2dbaf53d1fdda247c1d4264bba38e1a012c3e1eb3147c8", "top": "cac6a46d4d2223d9142b4633c8393cb055d902577e6fc5c82b8abb29b2d54b1e" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/original/front.png", "back": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/original/back.png", "left": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/original/left.png", "right": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/original/right.png", "top": "benchmarks/gallery-matrix/signal_lantern--gpt-6-sol--r03/plugin/original/top.png" }, "originalImageHashes": { "perspective": "f96382d0cb5a64eaad75c3abb0f9665219149f3c647041575c81fe9c691d6e8d", "front": "572a91af749890b4ed377f4f7f3f00261476feef89f5716e49788649f1d5db10", "back": "033ed9dae1f2b60228ba9dd7d28b536323db124401983fdbcb70902b9b868430", "left": "c27bf0ddf299093e5b24bc3c9019ad43e7cd5a80dd060241107b9a9b49bf24e2", "right": "48bc279082ddfbe45ea7b566ad117adc3b3864a55b593b9dc91e96288eac78e6", "top": "7e877193c4e107c48eeb16eb33972ca86d104fbbaa354fa7718e0c33a078e42c" } }, "criteria": [ { "id": "lantern_parts", "label": "lantern parts", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "lantern_handle_attachment", "label": "lantern handle attachment", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "lantern_glass_light", "label": "lantern glass light", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "lantern_finish", "label": "lantern finish", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } } ], "note": "The no-plugin result wins through richer manufactured construction, stronger rounded cap shaping, substantial handle fittings, and coherent secondary and tertiary detail across views. The plugin result satisfies every requirement but its simple flared body and conical cap feel less developed. The plugin result's flame is more immediately readable; the no-plugin result has the stronger overall finish.", "selection": "Run 3 selected by the gallery owner after viewing all three attempts. This is a curated example, not a random sample or an aggregate result." }, { "task": "joinery_stool", "taskTitle": "Splayed-leg joinery stool", "model": "gpt-6-astra", "modelTitle": "GPT 6 Astra", "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 373, "triangles": 15190, "images": { "perspective": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/hd/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/hd/bottom.png" }, "imageHashes": { "perspective": "6ea33b23348232cdf5f6d9dbd1b4e9ee6c35a2bb838589efc767d236af2468b0", "front": "d36f88e44ff2dc37a306f4a6643b6dace117fb210ec976e003ed86c2d3dace65", "back": "0dccacaaa3261bbf606b877a8d7a3b46d57a2fb01af544231b508b3b8d1494d5", "left": "35bf0bb95e042335e39a55578fe4e1123816b6fe309549bff81aac9b897a7955", "right": "afabfbcea8516fbe13db02b17c9f11315acc096d11333b4d98fa1829ef60fe02", "top": "5f8beef2c71d32bb7f88e2ba435709db3bf07171dffd4a55c4c55dce5de9b23d", "bottom": "279e2d87de879d828687ab603250ce163389790e49055b0776f8a233a3c29f40" }, "failedChecks": [], "artifactHashes": { "create_asset.py": "2d56c0631f2825688526afaff7f0673f92174f666aa7edd6f31dca9e71bb58af", "asset.blend": "63214a14ddb6cfa690a45d0dc0b9ae73462427b987bd4e5821b8113f69c33fec", "asset.glb": "57c2e8bcc9be031b0e8d30c1d4c1e89b73b8d6edde61570426c3db156c7bd04e" }, "generationTaskFingerprint": "28c3c6ad61d6de6e939213c55613d617296ab05d5b926a06521a998c7ea82fe8", "correctedTaskFingerprint": "2cbfc0d0f305e997e246fcd1d18761664f42e38cc53f721e487550d43101381f", "modelRequested": "gpt-6-astra", "modelsObserved": null, "agentVersion": "codex-cli 0.160.0", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": null, "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "baseline", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [ "Studio ground" ], "sourceSha256": "63214a14ddb6cfa690a45d0dc0b9ae73462427b987bd4e5821b8113f69c33fec", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/original/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/vanilla/original/bottom.png" }, "originalImageHashes": { "perspective": "e111cd03f3a1f2cbdb1f8d3cc205c9428337dfd08a9cbdf1d1d49e2e20aa70e3", "front": "7865c6aefcfd5982bb31f443d9edf70df767cc4b87218cf443b5d655f98f38a5", "back": "cad97edfa0dabb9cc0b5358039bb8e5dc6388ce53d9f5bccedba3bdc3655098c", "left": "81f0dd66301609fdc0d7bd2e84615235f8439c4b5dfc91a2edd4f2cde3394552", "right": "8449ab811bc7b26b85ef3d929e370d783be7f7fb77c87fa513a74516bc5503f9", "top": "f880942220524f004f7710e0c7aab20a7561aef749649dc45c507eb5c3e35f35", "bottom": "4202db62cb070834786aac0922c3071dd8f293ae10c749689752e31cb75f0a94" } }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 609, "triangles": 22620, "images": { "perspective": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/hd/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/hd/bottom.png" }, "imageHashes": { "perspective": "9082f34710440bcf20c18e9b35b4ff69a128718e4be1f6d8013c8eba192a19b9", "front": "969a5fb34f447189cad5a0ca6bd68687e4cadbd887201e75e54b1b5252528654", "back": "2f8b517c9ad55f4775648f3e68fe855d910a3f768a4ee579ed5b5c1ced8617a6", "left": "590f2e5870f0f53e6bfccb7b96e7e78bf0731b07de196b5708b348feeeb8562c", "right": "c7104145eff14d7077caabb1b9beb455b945aaa1619640094dea962919d2fd36", "top": "4c85df604424a619db7797b4a10df9a286b3faa2b63a782298dc2137c27d71cf", "bottom": "14253a7c6209f0f44b74f027e85d40f6e401808a150a1beec87474ab36f4e278" }, "failedChecks": [], "artifactHashes": { "create_asset.py": "4fee9c6d6289d9d15c1a4b8d9e400baef171583548980adf0cbf17548e262005", "asset.blend": "22927c5a2e6d7e9aa5cfaf40d3bbbc23d7161f2e22b7d845d6ecdf9f5d511fe6", "asset.glb": "1dfa5923a23e24957cf07922b287e67c2d0fb6de7370f9cf8268ca8b080c3e5a" }, "generationTaskFingerprint": "28c3c6ad61d6de6e939213c55613d617296ab05d5b926a06521a998c7ea82fe8", "correctedTaskFingerprint": "2cbfc0d0f305e997e246fcd1d18761664f42e38cc53f721e487550d43101381f", "modelRequested": "gpt-6-astra", "modelsObserved": null, "agentVersion": "codex-cli 0.160.0", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "skills", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "22927c5a2e6d7e9aa5cfaf40d3bbbc23d7161f2e22b7d845d6ecdf9f5d511fe6", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/original/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--gpt-6-astra--r01/plugin/original/bottom.png" }, "originalImageHashes": { "perspective": "980285d5a61e25c0a26a2c8aecde734c0f028e38d1c390c91628d2baaaf0a8f3", "front": "9e23d6374feb531c19a7763a50449f0076adf527750d361a829f4e74b87d1975", "back": "256fbcb3fc209925cce29d1a2871bd589773a2d6131ea77c66af26612ed9437b", "left": "b9189f5ef816d4b70506873d92967e5ca14caacd96a4d3dbd417df7c64e44841", "right": "b6b8e1975f99e7ce8a544a518263ac689ab5f8240009521ffaa5d5d4d0735261", "top": "43938137267b6cc52c9fdd0a77d856168696f681967a0068fffa7d0ac8278a41", "bottom": "c1652cab85c687496afb3de5b47225687cf91b8aac9bee8c5ec9e0ac002004ed" } }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "note": "One review noted: The plugin result provides a visibly finished stool with rounded edges, readable hardware and supported joint contacts across useful views. The no-plugin result shows the broad stool form, but its tiny framing and obstructed underside leave several critical construction requirements unverified.", "criteria": [ { "id": "seat_support", "label": "seat support", "question": "Do all four leg tops meet the seat underside without open seams or piercing its upper surface?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "stretcher_joints", "label": "stretcher joints", "question": "Do all eight stretcher ends meet their legs without floating gaps or exposed protruding ends?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "ground_support", "label": "ground support", "question": "Do all four feet terminate on the same plane without visibly hovering or being buried?", "critical": true, "vanilla": { "pass": 1, "fail": 0, "unclear": 1 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "negative_space", "label": "negative space", "question": "Are the spaces under the seat and inside the stretcher ring intentionally open and free of accidental crossing geometry?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "assembly_count", "label": "assembly count", "question": "Are exactly four legs and four stretchers visible across the views?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "finish", "label": "finish", "question": "Are curved edges, wood finish and inset hardware coherent on reverse sides and underside as well as the hero view?", "critical": false, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "rawReview": { "taskId": "joinery_stool", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "current", "current" ], "criterionMajorities": [ { "criterionId": "seat_support", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "stretcher_joints", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "ground_support", "baselinePasses": 1, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "negative_space", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "assembly_count", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "finish", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "seat_support", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "stretcher_joints", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "ground_support", "baselinePasses": 1, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "negative_space", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "assembly_count", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "current" }, "decodedWinner": "current", "scoresByMode": { "vanilla": { "taskFidelity": 5, "silhouetteAndProportion": 6, "constructionPlausibility": 5, "craftsmanshipAndDetail": 3, "materialsAndReadability": 4, "surfaceFinishAndShading": 5, "materialTextureQuality": 4, "lightingAndPresentation": 2, "finalStageCompleteness": 4, "multiviewConsistency": 5 }, "current": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.98, "scores": { "A": { "taskFidelity": 5, "silhouetteAndProportion": 6, "constructionPlausibility": 5, "craftsmanshipAndDetail": 3, "materialsAndReadability": 4, "surfaceFinishAndShading": 5, "materialTextureQuality": 4, "lightingAndPresentation": 2, "finalStageCompleteness": 4, "multiviewConsistency": 5 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "seat_support", "A": "unclear", "B": "pass", "evidenceA": "Small render scale prevents checking all four contacts; the underside tile is obscured by the floor.", "evidenceB": "Leg tops visibly meet the seat underside in the perspective and underside views, without visible gaps or upper-surface piercing." }, { "criterionId": "stretcher_joints", "A": "unclear", "B": "pass", "evidenceA": "Stretchers are present, but individual end contacts cannot be resolved reliably.", "evidenceB": "Perspective, opposing elevations and underside show the stretcher ends terminating against the legs without visible gaps or protrusions." }, { "criterionId": "ground_support", "A": "pass", "B": "pass", "evidenceA": "Feet align with the floor in the elevations, with no visible hovering or burial.", "evidenceB": "Opposing elevations show aligned foot bottoms and consistent floor contact." }, { "criterionId": "negative_space", "A": "unclear", "B": "pass", "evidenceA": "Elevations show open space, but the small perspective and blocked underside do not establish a clear ring interior.", "evidenceB": "The perspective and underside show an open ring interior and clear space beneath the seat, without accidental crossing members." }, { "criterionId": "assembly_count", "A": "unclear", "B": "pass", "evidenceA": "Four legs are discernible, but the images do not clearly separate all four stretchers.", "evidenceB": "Four legs and four perimeter stretchers are identifiable across the perspective and underside." }, { "criterionId": "finish", "A": "unclear", "B": "pass", "evidenceA": "Wood coloration is visible, but edge refinement, inset hardware and underside finish cannot be adequately inspected.", "evidenceB": "Rounded seat corners, softened edges, coherent wood surfaces and restrained recessed hardware remain consistent across opposing views and the underside." } ], "majorDefects": { "A": [ "Excessive framing distance makes the asset and its joints difficult to inspect.", "The underside tile shows the floor instead of the required construction evidence.", "Hardware and polished edge treatment are not legible." ], "B": [] }, "rationale": "B provides a visibly finished stool with rounded edges, readable hardware and supported joint contacts across useful views. A shows the broad stool form, but its tiny framing and obstructed underside leave several critical construction requirements unverified." } }, { "mapping": { "A": "current", "B": "vanilla" }, "decodedWinner": "current", "scoresByMode": { "current": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8.5, "multiviewConsistency": 8.5 }, "vanilla": { "taskFidelity": 4.5, "silhouetteAndProportion": 6, "constructionPlausibility": 4, "craftsmanshipAndDetail": 3, "materialsAndReadability": 4, "surfaceFinishAndShading": 4, "materialTextureQuality": 3, "lightingAndPresentation": 2, "finalStageCompleteness": 4, "multiviewConsistency": 4 } }, "result": { "winner": "A", "confidence": 0.98, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8.5, "multiviewConsistency": 8.5 }, "B": { "taskFidelity": 4.5, "silhouetteAndProportion": 6, "constructionPlausibility": 4, "craftsmanshipAndDetail": 3, "materialsAndReadability": 4, "surfaceFinishAndShading": 4, "materialTextureQuality": 3, "lightingAndPresentation": 2, "finalStageCompleteness": 4, "multiviewConsistency": 4 } }, "criterionResults": [ { "criterionId": "seat_support", "A": "pass", "B": "unclear", "evidenceA": "Leg tops meet the seat underside in the perspective and underside views; no open seams or upper-surface piercing are visible.", "evidenceB": "The stool is too small to resolve seat contacts, and the underside tile is obscured by a large plane." }, { "criterionId": "stretcher_joints", "A": "pass", "B": "unclear", "evidenceA": "The perspective and underside views expose the four-sided ring with ends meeting the legs without visible gaps or protrusions.", "evidenceB": "Some stretchers are visible, but all eight end contacts cannot be inspected at this scale." }, { "criterionId": "ground_support", "A": "pass", "B": "unclear", "evidenceA": "Feet align with the ground across the opposing elevations without visible hovering or burial.", "evidenceB": "The small feet and bright platform prevent reliable inspection of contact and burial." }, { "criterionId": "negative_space", "A": "pass", "B": "unclear", "evidenceA": "Elevations and perspective show open space beneath the seat and through the ring, with no accidental crossing members.", "evidenceB": "Elevations suggest open space, but the ring interior and underside are insufficiently exposed to verify clearance." }, { "criterionId": "assembly_count", "A": "pass", "B": "unclear", "evidenceA": "Four legs and four perimeter stretchers are identifiable across perspective and underside views.", "evidenceB": "Four legs are apparent, but the complete four-stretcher count is not securely readable." }, { "criterionId": "finish", "A": "pass", "B": "unclear", "evidenceA": "Rounded seat corners, softened member edges, wood shading and restrained inset fasteners remain coherent across the views.", "evidenceB": "Wood coloration is visible, but edge refinement, inset hardware and underside finish cannot be assessed." } ], "majorDefects": { "A": [], "B": [ "Excessive framing around a large platform makes the asset too small for joint and finish inspection.", "The underside tile shows an opaque plane instead of the stool's underside." ] }, "rationale": "A visibly demonstrates a finished stool with readable joints, a complete stretcher ring and coherent rounded detailing. B's distant framing and obscured underside leave most critical construction requirements unverified." } } ] }, "id": "joinery_stool--gpt-6-astra--r01", "repetition": 1, "cohort": "October 2026 spatial pilot", "limitation": "One generation per condition; two counterbalanced Astra judgments of the same images. Vanilla runs were collected later and paired with the existing frozen plugin runs; generation order was not counterbalanced. With plugin means pinned skill workflows and bundled scripts, not an MCP ablation. Concurrent times are not a latency benchmark. Structural scores are proxies, not visual quality or proof of contact.", "judgeCount": 2 }, { "task": "task_lamp_clearance_holdout", "taskTitle": "Articulated task-lamp clearance holdout", "model": "gpt-6-astra", "modelTitle": "GPT 6 Astra", "vanilla": { "score": 91.83, "rawScore": 91.83, "hardGate": false, "rawHardGate": false, "seconds": 675, "triangles": 38520, "images": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/hd/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/hd/bottom.png" }, "imageHashes": { "perspective": "d8c5737d8170e7be89ec0db8003f41129d43a6ba8120356e3f4c8078ee7d3f60", "front": "4d44b0367e3d6d64d200e7ccbed77974abd7d84c726d567c87ba5856f418cf6d", "back": "d9bab3b67d513fc5bbe3b45d4561bf701d0ebe2ca4270a8a827b0496b952c9fc", "left": "a79c002f79f26d6a9d88a7f81bdc7196e5236351ccf09728eaa4c5cf8f1f8c9e", "right": "62dec66f4987ee183d4c6b50ea633247a33c7033a6ccbf3269dd5a87d66b01f0", "top": "3686b7c74d53814e45b011958d19221c860860b5275e86848bced0e4220a9841", "bottom": "d48449d51a404d86e23ad843dbb92c514b66919257555b58bc2a302afd884ba7" }, "failedChecks": [ { "id": "semantic_part_coverage", "passed": false, "earned": 20.833333333333336, "possible": 25, "detail": "5/6 required semantic part groups found" }, { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "24 invalid vertices, degenerate faces, or zero-length edges" } ], "artifactHashes": { "create_asset.py": "ce7753280fa5ac88877c3aa038ba2a13425496e0100f1f644cbe2f447d4f0f44", "asset.blend": "4b032fb902d525c57bb62124a8c81f9feb102e79880cb3814b9f15e5484aa8df", "asset.glb": "2cc52015bcc28b432182ab6a3e28475d39f1aa6fc65641e8f180c8a5dc301992" }, "generationTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "correctedTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "modelRequested": "gpt-6-astra", "modelsObserved": null, "agentVersion": "codex-cli 0.160.0", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": null, "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "baseline", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [ "Studio | compact curved backdrop" ], "sourceSha256": "4b032fb902d525c57bb62124a8c81f9feb102e79880cb3814b9f15e5484aa8df", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/original/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/vanilla/original/bottom.png" }, "originalImageHashes": { "perspective": "ff0760adf6216668c23702261af2cb234b142cc32b0b78c64a7298b3d14889bf", "front": "67ac1145d3efb09136895c7eae0ac945161a5575c5e45cea575a39761dfbf243", "back": "eb0463c9da19100b21a37e4536c5da426675a82d89890b51f663441b8f154c72", "left": "6ced1f572b4e8b33a58c0751186bc96f6a714cff719b24f8e5972f9b9f9eba46", "right": "63e07f77389cfc9ee384b93c691db739fe3d8d1ef0913bb074e0dc94f494b61e", "top": "333c147f78de07e384c23715d8a07c34560d8924913c8f1e218e13d5e5f715cf", "bottom": "9fc9a72a31fb232754d050bc7b8b571e44014ba431a365f36c8644b9e0aec211" } }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 692, "triangles": 34428, "images": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/hd/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/hd/bottom.png" }, "imageHashes": { "perspective": "1351dc1dff595a40ab195689bfe4fb3da17ecd8f692882422d7cd4cedfeb2504", "front": "78f22580df53584cd43cc9bf08de30e3530e71879d14960d59d6f3a37d4c3021", "back": "84866119381a1ac681e3f269c1a12c69e08a5d75bea7323a759ac438a409c807", "left": "d9dd12d1f167eae10242c433afa3188e04cab5f81540904e9c96e9ac37ac61de", "right": "643100638821f86b1924cbdc66c7ab78f631a381fa3eec0df61d627d7c100ce9", "top": "fd18293ba3bd2b6d987aa19490ad2e1382fb2e2bf94431c4ebba0be295c0e94c", "bottom": "a7823bc48c9096a9b3d4c5c51e6ebf1c90b45c0ea6374aaaf802de4eddb769ad" }, "failedChecks": [], "artifactHashes": { "create_asset.py": "56fb5f706b1f0e0352c385e675df48d364d6df26f212ee57b223f8eb1ef958e5", "asset.blend": "ed262c2bee59de016d1597a702c2e057b0a90727a029fcbf4c9e7e8baa66122f", "asset.glb": "65d831c2d8ed47026064bdca6bd316ff8b910e2e0512403688468b4d9d210c6b" }, "generationTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "correctedTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "modelRequested": "gpt-6-astra", "modelsObserved": null, "agentVersion": "codex-cli 0.160.0", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "skills", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "ed262c2bee59de016d1597a702c2e057b0a90727a029fcbf4c9e7e8baa66122f", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/original/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6-astra--r01/plugin/original/bottom.png" }, "originalImageHashes": { "perspective": "d60cc2693396d8825c016aa145d503d48b8919c8bf203d5a0ef7be602e3681d8", "front": "c34306084b3697f456edccf8176184c6d0721ec21bed1c42e176777bc7cf71e3", "back": "36678e63df86de541fcdad54b69e6a6232f183c4d9b717dd60edcc8becd1a321", "left": "10b6d6d2f0ed3897109e5565eef1f11d303f9cf2244e149272f868c33ab21d14", "right": "86ee311a661eeb7f036ae394deb965eff8a0b12ca7095364accfd25016db74af", "top": "1692cd3b9a1b6a7f7c88e912e20be8211a085c991792b946888e4eabe7dccf2a", "bottom": "57e63b3dc681834f527d21643e4c9531dbe3925ad88b72d6ee5927c9a365e462" } }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "note": "One review noted: The plugin result provides clear surface evidence of connected hinges, mechanical gaps, a recessed bulb, seated cable, and finished underside construction. The no-plugin result's distant framing and obstructing backdrop prevent verification of most critical relationships.", "criteria": [ { "id": "joint_continuity", "label": "joint continuity", "question": "Are base-to-arm, elbow, and arm-to-shade connections physically continuous through credible hinge hardware?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "hinge_clearance", "label": "hinge clearance", "question": "Do adjacent arms and hinge cheeks retain readable mechanical clearance without unintended intersections?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "shade_cavity", "label": "shade cavity", "question": "Is the shade mouth open with a distinct recessed bulb that does not pierce the shell?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cable_contact", "label": "cable contact", "question": "Does the cable meet its socket and rest on the desk plane without floating or being buried?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "base_contact", "label": "base contact", "question": "Does the weighted base sit flat on the supporting plane?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "finish", "label": "finish", "question": "Do enamel, brushed metal, rubber and reverse-side construction details read as a finished product?", "critical": false, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "rawReview": { "taskId": "task_lamp_clearance_holdout", "repetition": 1, "baselineAutomatedScore": 91.83, "candidateAutomatedScore": 100, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "current", "current" ], "criterionMajorities": [ { "criterionId": "joint_continuity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "hinge_clearance", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "shade_cavity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cable_contact", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "base_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "finish", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "joint_continuity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "hinge_clearance", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "shade_cavity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cable_contact", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "base_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "current" }, "decodedWinner": "current", "scoresByMode": { "vanilla": { "taskFidelity": 4, "silhouetteAndProportion": 5, "constructionPlausibility": 4, "craftsmanshipAndDetail": 3, "materialsAndReadability": 4, "surfaceFinishAndShading": 5, "materialTextureQuality": 4, "lightingAndPresentation": 2, "finalStageCompleteness": 4, "multiviewConsistency": 3 }, "current": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.98, "scores": { "A": { "taskFidelity": 4, "silhouetteAndProportion": 5, "constructionPlausibility": 4, "craftsmanshipAndDetail": 3, "materialsAndReadability": 4, "surfaceFinishAndShading": 5, "materialTextureQuality": 4, "lightingAndPresentation": 2, "finalStageCompleteness": 4, "multiviewConsistency": 3 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "joint_continuity", "A": "unclear", "B": "pass", "evidenceA": "Arm segments appear connected, but their tiny presentation prevents verification of all hinge interfaces.", "evidenceB": "Base pivot, circular elbow hardware, and shade yoke visibly connect the articulated components." }, { "criterionId": "hinge_clearance", "A": "unclear", "B": "pass", "evidenceA": "Hinge cheeks and adjacent surfaces are too small to establish clearance.", "evidenceB": "Front and overhead views reveal separated arm rails, hinge spacers, and shade brackets with readable gaps." }, { "criterionId": "shade_cavity", "A": "unclear", "B": "pass", "evidenceA": "The light-colored shade mouth does not resolve into independently readable shell and recessed bulb.", "evidenceB": "Front and underside views show an open shade, distinct interior, and recessed bulb contained within the rim." }, { "criterionId": "cable_contact", "A": "unclear", "B": "pass", "evidenceA": "A cable loop is visible, but socket seating and desk contact cannot be established at this scale.", "evidenceB": "Cable visibly enters the base fitting and trails along the supporting plane to the plug." }, { "criterionId": "base_contact", "A": "pass", "B": "pass", "evidenceA": "Visible side views place the round base against the supporting surface without a discernible gap.", "evidenceB": "Side views show the base seated level with its lower supports contacting the desk." }, { "criterionId": "finish", "A": "unclear", "B": "pass", "evidenceA": "Glossy teal surfaces are apparent, but material distinctions and reverse-side construction are obscured or too small.", "evidenceB": "Smooth enamel, contrasting metallic hardware, dark cable, shade interior, and underside plate with feet read as a finished product." } ], "majorDefects": { "A": [ "Oversized backdrop dominates framing and makes the lamp too small for critical surface inspection.", "Backdrop completely obscures the lamp in the back and underside tiles.", "Hinge clearance, bulb recess, cable seating, and reverse-side finish lack sufficient visible evidence." ], "B": [] }, "rationale": "B provides clear surface evidence of connected hinges, mechanical gaps, a recessed bulb, seated cable, and finished underside construction. A's distant framing and obstructing backdrop prevent verification of most critical relationships." } }, { "mapping": { "A": "current", "B": "vanilla" }, "decodedWinner": "current", "scoresByMode": { "current": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 4, "silhouetteAndProportion": 5, "constructionPlausibility": 4, "craftsmanshipAndDetail": 3, "materialsAndReadability": 4, "surfaceFinishAndShading": 5, "materialTextureQuality": 3, "lightingAndPresentation": 1, "finalStageCompleteness": 3, "multiviewConsistency": 3 } }, "result": { "winner": "A", "confidence": 0.99, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "B": { "taskFidelity": 4, "silhouetteAndProportion": 5, "constructionPlausibility": 4, "craftsmanshipAndDetail": 3, "materialsAndReadability": 4, "surfaceFinishAndShading": 5, "materialTextureQuality": 3, "lightingAndPresentation": 1, "finalStageCompleteness": 3, "multiviewConsistency": 3 } }, "criterionResults": [ { "criterionId": "joint_continuity", "A": "pass", "B": "unclear", "evidenceA": "Visible axle hardware connects the base bracket, elbow, and shade yoke continuously.", "evidenceB": "Connections are too small to verify credible hardware across the required joints." }, { "criterionId": "hinge_clearance", "A": "pass", "B": "unclear", "evidenceA": "Frontal and overhead views reveal spaced rails and hinge cheeks with readable separation around the pivots.", "evidenceB": "The distant framing does not resolve gaps between arms and hinge cheeks." }, { "criterionId": "shade_cavity", "A": "pass", "B": "unclear", "evidenceA": "Front and underside views expose an open reflector interior and a distinct inset bulb without visible shell penetration.", "evidenceB": "A light-colored mouth is visible, but bulb recession and shell clearance cannot be established." }, { "criterionId": "cable_contact", "A": "pass", "B": "unclear", "evidenceA": "The cable visibly enters the base fitting and descends onto the desk, where its trailing length and plug rest.", "evidenceB": "The cable is visible in several views, but socket seating and surface contact are too small to inspect." }, { "criterionId": "base_contact", "A": "pass", "B": "pass", "evidenceA": "Side and frontal views show a level base with its lower edge seated against the supporting surface.", "evidenceB": "Visible side views show the base seated on the backdrop's flat supporting section without an apparent gap." }, { "criterionId": "finish", "A": "pass", "B": "unclear", "evidenceA": "Smooth teal enamel, contrasting metal hardware, dark cable, and an underside plate with feet read as finished product details.", "evidenceB": "Teal gloss and pale hardware are visible, but material refinement and reverse-side construction are not adequately exposed." } ], "majorDefects": { "A": [], "B": [ "Oversized backdrop dominates the composition and reduces the lamp to an inspection-inadequate scale.", "The backdrop completely occludes the lamp in the back and underside tiles.", "Critical hinge, bulb, cable, and reverse-side details remain unverifiable." ] }, "rationale": "A provides clear surface-level evidence of articulated construction, shade depth, cable seating, and finished underside details. B's distant framing and obstructing backdrop prevent verification of most critical requirements." } } ] }, "id": "task_lamp_clearance_holdout--gpt-6-astra--r01", "repetition": 1, "cohort": "October 2026 spatial pilot", "limitation": "One generation per condition; two counterbalanced Astra judgments of the same images. Vanilla runs were collected later and paired with the existing frozen plugin runs; generation order was not counterbalanced. With plugin means pinned skill workflows and bundled scripts, not an MCP ablation. Concurrent times are not a latency benchmark. Structural scores are proxies, not visual quality or proof of contact.", "judgeCount": 2 }, { "id": "tabletop_press--gpt-6-astra--r01", "task": "tabletop_press", "taskTitle": "Tabletop lever press", "model": "gpt-6-astra", "modelTitle": "GPT 6 Astra", "repetition": 1, "cohort": "September 2026 quality study", "judgeCount": 3, "limitation": "Historical skill-workflow comparison, with the original model and evaluator. Three reviews of each pair. Scores are kept separate from the October pilot. Some judged views hide oversized studio floors; original delivery failures are retained.", "votes": { "baseline": 0, "candidate": 3, "tie": 0 }, "rawReview": { "taskId": "tabletop_press", "repetition": 1, "baselineAutomatedScore": 94, "candidateAutomatedScore": 100, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "skills", "skills", "skills" ], "criterionMajorities": [ { "criterionId": "press_parts", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "press_drive_chain", "baselinePasses": 2, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "press_supports", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "press_finish", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 } ], "criticalCriterionMajorities": [ { "criterionId": "press_parts", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "press_drive_chain", "baselinePasses": 2, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "press_supports", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "skills", "B": "baseline" }, "decodedWinner": "skills", "scoresByMode": { "skills": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "baseline": { "taskFidelity": 8.4, "silhouetteAndProportion": 8.1, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8.4, "finalStageCompleteness": 8.2, "multiviewConsistency": 8.8 } }, "result": { "winner": "A", "confidence": 0.82, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.4, "silhouetteAndProportion": 8.1, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8.4, "finalStageCompleteness": 8.2, "multiviewConsistency": 8.8 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "All required parts are visible, including articulated linkage, exposed spring, aligned tooling, and side tray.", "evidenceB": "All required parts are visible, including a compact lever follower, spring, ram, die, and supported tray." }, { "criterionId": "press_drive_chain", "A": "pass", "B": "pass", "evidenceA": "Side views clearly connect the rear lever pivot through a hinged link to the spring-topped ram above the die.", "evidenceB": "The pivoted lever connects to the ram through a small collar/follower above the spring; ram and die align, although articulation is less explicit." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "Spring seats, pivot supports, ram housing, die platform, and tray rails provide visible support.", "evidenceB": "Spring is seated over the housing, tooling rests on the platform, and the tray has visible supporting rails." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Beveled cast-style frame, inset panels, fasteners, ribbed grip, and separated metal finishes read as manufactured.", "evidenceB": "Smooth enamel surfaces, beveled edges, metallic tooling, fasteners, and grip detailing establish a finished industrial prop." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "A wins through a more legible articulated drive, stronger frame proportions, and richer manufactured detailing across all views. B is coherent and finished, but its tiny follower connection and simpler frame provide less mechanical clarity and refinement." } }, { "mapping": { "A": "baseline", "B": "skills" }, "decodedWinner": "skills", "scoresByMode": { "baseline": { "taskFidelity": 8, "silhouetteAndProportion": 7.7, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.3, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 8, "lightingAndPresentation": 8.3, "finalStageCompleteness": 8, "multiviewConsistency": 8.7 }, "skills": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.8, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.3, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.87, "scores": { "A": { "taskFidelity": 8, "silhouetteAndProportion": 7.7, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.3, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 8, "lightingAndPresentation": 8.3, "finalStageCompleteness": 8, "multiviewConsistency": 8.7 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.8, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.3, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "Base, C-frame, pivoted lever, small upper coupling, ram, die, coil spring, and shallow tray are visible.", "evidenceB": "All required parts are visible, including distinct pivot hardware, paired linkage members, return spring, ram, die, and collection tray." }, { "criterionId": "press_drive_chain", "A": "pass", "B": "pass", "evidenceA": "The lever connects near the spring-topped ram through a compact coupling; the ram is aligned over the die. Joint articulation is less clearly resolved.", "evidenceB": "Side views clearly trace the rear lever pivot through a pinned link to the spring-topped ram, aligned over the die." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "The spring seats above the head, the ram passes through its guide, and the tray rests on a visible base-supported structure.", "evidenceB": "The spring has visible seating, pivot and linkage have supporting hardware, and projecting rails support the side tray." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Beveled teal housing, metallic tooling, ribbed grip, fasteners, and labels establish a finished manufactured appearance.", "evidenceB": "Rounded housing edges, inset frame detailing, pinned orange linkage, machined tooling, fastened plates, and a ribbed grip provide coherent industrial refinement." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins through a more legible pinned drive mechanism, sturdier frame proportions, and more developed manufactured detailing across the views. A is also finished and coherent, but its compact upper coupling and simpler frame make the mechanism less convincing." } }, { "mapping": { "A": "skills", "B": "baseline" }, "decodedWinner": "skills", "scoresByMode": { "skills": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.5, "lightingAndPresentation": 8.6, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "baseline": { "taskFidelity": 8, "silhouetteAndProportion": 8.1, "constructionPlausibility": 7.3, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.2, "multiviewConsistency": 8.8 } }, "result": { "winner": "A", "confidence": 0.86, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.5, "lightingAndPresentation": 8.6, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8, "silhouetteAndProportion": 8.1, "constructionPlausibility": 7.3, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.2, "multiviewConsistency": 8.8 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "Base, substantial frame, pivoted lever, articulated linkage, ram, die, exposed spring, and side tray are visible.", "evidenceB": "All required components are visible, including a small lever-to-ram connector and exposed return spring." }, { "criterionId": "press_drive_chain", "A": "pass", "B": "unclear", "evidenceA": "Side views show a distinct pivoted link connecting the lever to the spring-topped ram, aligned over the die.", "evidenceB": "Ram and die align, but the small connection beneath the straight lever does not clearly explain how lever rotation accommodates vertical ram travel." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "Spring seats against the ram housing; tray has visible mounting rails, and the die rests on a supported platen.", "evidenceB": "Spring is seated on the upper housing, tray supports connect toward the base, and the die platform is supported by the frame." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded painted castings, metallic working surfaces, ribbed grip, fasteners, and inset plates create a refined manufactured appearance.", "evidenceB": "Smooth painted surfaces, beveled edges, contrasting metal hardware, grip ribs, and labels provide a finished industrial appearance." } ], "majorDefects": { "A": [], "B": [ "Lever-to-ram articulation is insufficiently resolved visually to establish a plausible moving drive chain." ] }, "rationale": "A wins through its clearly articulated drive linkage, more substantial frame, and richer manufacturing detail. Both have polished surfaces and consistent views, but B's simplified lever connection leaves the central mechanism less convincing." } } ] }, "vanilla": { "score": 94, "rawScore": 94, "hardGate": false, "rawHardGate": false, "seconds": 303, "triangles": 21446, "images": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/hd/top.png" }, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "115 invalid vertices, degenerate faces, or zero-length edges" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "200.000m maximum extent; limit 3m" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "agentVersion": "codex-cli 0.155.0-alpha.9.2", "evaluatorFingerprint": "07a858a97099d561a6161b825aaac0de8fc8e1cc84a73194cbb121bb97cee695", "generationTaskFingerprint": "e147da73ad0c4c84e79c127ae9eb1e37b1a299e26700ddfb96d17df5c0172260", "rawScorerVersion": 5, "modelRequested": "gpt-6-astra", "sourceHashesRecordedAtGalleryBuild": true, "artifactHashes": { "asset.blend": "1d81ea67602d1eb83f9ef8420963bb10841430ad57649dc6377ee829d01b85d1", "asset.glb": "e4de7e57865ed6eb7e467211d762196695adc829d6b4c08efa465f54d91d1517" }, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [ "Studio ground" ], "sourceSha256": "1d81ea67602d1eb83f9ef8420963bb10841430ad57649dc6377ee829d01b85d1", "sourceModified": false }, "imageHashes": { "perspective": "f0ddb4781dc9be964555ae9d6dec49afe6d61764b971c07ad53f6f6ea2a333be", "front": "0df0a391892e552acd3f6a6fda66a4232c82b646b85061148c63302e9c15951e", "back": "51d3ff4124f445526e332ed81f2e734c3d6c20805ed9753a9a75f6ac296e740d", "left": "2392a9ecace14ea82adbc840330730f7aa83da4d921e42fdf467ec4f1cafaebd", "right": "adbe3146f9ab2384569a441f996eda4e6e624fdf5f17c71fa99dd24984859e8d", "top": "5bbb7778cacf7437f123373546282bc592ae64c1d84d8b0a2dd52ecfc39c87b7" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "ce6fd5a7d3c1e4f290115624c2bbf0b9228d02a9de4b78c4d6daf0b0a31dc7ea", "front": "c41a6399daaa32558e9b00acaa15df251559046e50f5783019ef94f641b747a0", "back": "5b04f3387dbce31888cf43e4bc8111ed8b86900fc2f72fb9dbba0b26dc6c6d64", "left": "cf1c276ed4e86877a0654877709c0b88d9a7a45411cf3115c3ba7186313646ac", "right": "9dbfb7db1e810515775ce0ad3f0565389f678309c6a02c0bbf36dbf49dd1de8b", "top": "3e607b861848e8bb17ed07d8f025dadc995b85c38f0b15d04dae3c55cbab146a" } }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 709, "triangles": 23540, "images": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/hd/top.png" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fd73795a211744fff6462d9ed0b38296bb8b9c01e44ad7ca5a6f3a232b7deec7", "agentVersion": "codex-cli 0.155.0-alpha.9.2", "evaluatorFingerprint": "07a858a97099d561a6161b825aaac0de8fc8e1cc84a73194cbb121bb97cee695", "generationTaskFingerprint": "e147da73ad0c4c84e79c127ae9eb1e37b1a299e26700ddfb96d17df5c0172260", "rawScorerVersion": 5, "modelRequested": "gpt-6-astra", "sourceHashesRecordedAtGalleryBuild": true, "artifactHashes": { "asset.blend": "71d25efd6deef460fe16e0fa0eebc0ab30905a7d3a0973409bc6cbb3177e5b98", "asset.glb": "2dc41d7a91d252dd88f092a24f6ae4f9e4aba1dc2cda31c41a0eb18c2d3ec207" }, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "71d25efd6deef460fe16e0fa0eebc0ab30905a7d3a0973409bc6cbb3177e5b98", "sourceModified": false }, "imageHashes": { "perspective": "ac1aae998d29a7003e8a1865a4bdf4638ce510d4ddbe9b29288126287ba9407a", "front": "4c5e7f61ae9f234c0ccb800621c82f2990370cf1c32c301b6052fb42ab81f569", "back": "8cd394ddfae985c98c7003384ab7c4c5ae4cc5cacc7475de5bb4621a1d74f876", "left": "b898b749f33979b3bf8f98dce920f2e75f89f3032736e4907aca2aa99f5043e6", "right": "61cc79e8ef5b52a686bb7cc2cf2bf939e5d17bae476f496fe0251e18430a2d45", "top": "57dc2e7903416bdf68dfa5211f1c7c2b455a20bf2d892f714e186c2b00079f54" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6-astra--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "ac4d6447cb168efb7cea50f08ae37be4835babbfc69b265899f9251603923f41", "front": "39a63289f165590b593dbfa24b9f70674fa123c422ff2324bd1625799802f392", "back": "1fe74d7f8e0d56abda613626879d49a2c97405bb98fab6ee979489a287b764ec", "left": "be3456fb7b33ffe1b88fa95e3bd29bb74dec12a6549ecbc8d4c63ef61d2cc873", "right": "ac9b92bbb3d63e27a0e7909c5471f436664bd6429206a62bef7fcf5b5c005878", "top": "3ff073d3faa4694d2534b5264ea92191f21f8f347cc41b47e5075236132bb68c" } }, "criteria": [ { "id": "press_parts", "label": "press parts", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "press_drive_chain", "label": "press drive chain", "vanilla": { "pass": 2, "fail": 0, "unclear": 1 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "press_supports", "label": "press supports", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "press_finish", "label": "press finish", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } } ], "note": "The plugin result wins through a more legible articulated drive, stronger frame proportions, and richer manufactured detailing across all views. The no-plugin result is coherent and finished, but its tiny follower connection and simpler frame provide less mechanical clarity and refinement." }, { "id": "winch_drawbridge--gpt-6-astra--r01", "task": "winch_drawbridge", "taskTitle": "Animated miniature winch drawbridge", "model": "gpt-6-astra", "modelTitle": "GPT 6 Astra", "repetition": 1, "cohort": "September 2026 quality study", "judgeCount": 3, "limitation": "Historical skill-workflow comparison, with the original model and evaluator. Three reviews of each pair. Scores are kept separate from the October pilot. Some judged views hide oversized studio floors; original delivery failures are retained. This gallery shows still views; the historical drawbridge review also used sampled animation frames.", "votes": { "baseline": 0, "candidate": 3, "tie": 0 }, "rawReview": { "taskId": "winch_drawbridge", "repetition": 1, "baselineAutomatedScore": 90, "candidateAutomatedScore": 100, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "skills", "skills", "skills" ], "criterionMajorities": [ { "criterionId": "bridge_parts", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "bridge_hinge", "baselinePasses": 2, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "bridge_cables", "baselinePasses": 0, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "bridge_finish", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 } ], "criticalCriterionMajorities": [ { "criterionId": "bridge_parts", "baselinePasses": 3, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "bridge_hinge", "baselinePasses": 2, "candidatePasses": 3, "judgeCount": 3 }, { "criterionId": "bridge_cables", "baselinePasses": 0, "candidatePasses": 3, "judgeCount": 3 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "skills", "B": "baseline" }, "decodedWinner": "skills", "scoresByMode": { "skills": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "baseline": { "taskFidelity": 7.8, "silhouetteAndProportion": 7.7, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 8.2, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 7.7, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "result": { "winner": "A", "confidence": 0.88, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.8, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.8, "multiviewConsistency": 9 }, "B": { "taskFidelity": 7.8, "silhouetteAndProportion": 7.7, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 8.2, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 7.7, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two braced supports, crossbeam, segmented deck hinge, drum assembly, crank, and paired cables are visible.", "evidenceB": "Two substantial pillars, crossbeam, deck, lower pivot hardware, drum, crank, and paired cables are visible." }, { "criterionId": "bridge_hinge", "A": "pass", "B": "pass", "evidenceA": "The three poses retain the deck root at the visible lower hinge while the deck rises into the opening.", "evidenceB": "The deck root remains at the lower pivot across the three poses while its free end rises above the crossbeam." }, { "criterionId": "bridge_cables", "A": "pass", "B": "unclear", "evidenceA": "Paired drum windings and deck corner fittings support plausible cable connections; exposed spans shorten as the deck approaches the drum.", "evidenceB": "Both cables connect plausibly in the lowered pose, but the raised deck and crossbeam obscure the routes and upper deck connections sufficiently to prevent verification." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Consistent softened edges, timber coloration, metal straps, fasteners, and hinge detailing remain coherent across the sheets.", "evidenceB": "Smooth bevels, layered masonry bases, turquoise trim, and warm deck surfaces remain consistent across views and poses." } ], "majorDefects": { "A": [], "B": [ "The raised deck projects above the crossbeam, weakening the fit between the moving deck and support frame.", "Cable continuity cannot be verified in the raised pose because key routes and connections are obscured." ] }, "rationale": "A wins through clearer mechanical articulation, braced supports, richer functional detailing, and a raised deck that fits its frame. B has clean surface refinement but less convincing deck-to-frame proportions and weaker raised-state cable evidence. Neither sheet establishes continuous timing, loop smoothness, flicker absence, or between-frame collision clearance." } }, { "mapping": { "A": "baseline", "B": "skills" }, "decodedWinner": "skills", "scoresByMode": { "baseline": { "taskFidelity": 7.5, "silhouetteAndProportion": 7.8, "constructionPlausibility": 6.7, "craftsmanshipAndDetail": 7.4, "materialsAndReadability": 8, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 7.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 7.6, "multiviewConsistency": 8 }, "skills": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.8, "multiviewConsistency": 8.7 } }, "result": { "winner": "B", "confidence": 0.89, "scores": { "A": { "taskFidelity": 7.5, "silhouetteAndProportion": 7.8, "constructionPlausibility": 6.7, "craftsmanshipAndDetail": 7.4, "materialsAndReadability": 8, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 7.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 7.6, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.8, "multiviewConsistency": 8.7 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two capped supports, planked deck, crossbeam, wound drum, side crank and paired cables are visible.", "evidenceB": "All required components are visible, with additional braces, axle housings and deck attachment hardware." }, { "criterionId": "bridge_hinge", "A": "unclear", "B": "pass", "evidenceA": "The three poses suggest rotation around a consistent deck root, but the substantial masonry bases obscure a clearly identifiable hinge.", "evidenceB": "Visible hinge barrels and side pivot hardware establish the deck root; lowered, halfway and raised poses retain that pivot location." }, { "criterionId": "bridge_cables", "A": "unclear", "B": "pass", "evidenceA": "Paired cables connect the drum region and deck corners when lowered. Occlusion prevents verification of both connections in the halfway and raised poses.", "evidenceB": "Paired drum windings, cables and deck corner eyelets form plausible connections across the shown poses, with the raised eyelets approaching the drum." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded stone tiers, softened timber edges and consistent teal and brass accents maintain a polished finish across views.", "evidenceB": "Beveled timber, metal bands, fasteners, hinge hardware and consistent material treatment remain coherent across views and poses." } ], "majorDefects": { "A": [ "Deck hinge construction is insufficiently exposed to verify the requested visible pivot.", "Both cable connections cannot be verified in the elevated poses." ], "B": [] }, "rationale": "B provides stronger visible engineering through braced supports, explicit hinges, axle housings and cable eyelets. A is polished but its bulky bases obscure the pivot and its elevated cable connections are less readable. Continuous timing, loop smoothness, flicker and between-frame collisions remain unclear for both." } }, { "mapping": { "A": "skills", "B": "baseline" }, "decodedWinner": "skills", "scoresByMode": { "skills": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 8.7, "materialsAndReadability": 8.6, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.1, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.7, "multiviewConsistency": 8.7 }, "baseline": { "taskFidelity": 7.8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.4, "craftsmanshipAndDetail": 7.9, "materialsAndReadability": 8.2, "surfaceFinishAndShading": 8.6, "materialTextureQuality": 7.7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8, "multiviewConsistency": 8.1 } }, "result": { "winner": "A", "confidence": 0.84, "scores": { "A": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 8.7, "materialsAndReadability": 8.6, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.1, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.7, "multiviewConsistency": 8.7 }, "B": { "taskFidelity": 7.8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.4, "craftsmanshipAndDetail": 7.9, "materialsAndReadability": 8.2, "surfaceFinishAndShading": 8.6, "materialTextureQuality": 7.7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8, "multiviewConsistency": 8.1 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two braced posts, crossbeam, segmented deck hinge, twin cable windings, axle and external crank are visible.", "evidenceB": "Two substantial posts, crossbeam, deck pivot hardware, drum, crank and two lifting cables are visible." }, { "criterionId": "bridge_hinge", "A": "pass", "B": "pass", "evidenceA": "The three poses retain the deck's lower edge at the visible hinge while its free edge rises into the opening.", "evidenceB": "The deck progresses from horizontal through inclined to upright around the same lower pivot location." }, { "criterionId": "bridge_cables", "A": "pass", "B": "unclear", "evidenceA": "Paired cables reach the deck corners when lowered and shorten toward the visible upper attachment rings as the deck rises; sampled connections appear plausible.", "evidenceB": "Both cables are readable when lowered, but the raised deck extends above the beam and obscures the routing. Both drum-to-deck connections cannot be verified in the final pose." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Consistent softened edges, timber coloration, teal fittings and brass fasteners carry across views and poses without obvious blockout residue.", "evidenceB": "Smooth bevels, pale timber, layered masonry bases and coordinated metal accents remain consistent across the supplied views and poses." } ], "majorDefects": { "A": [], "B": [ "Raised-state cable routing is insufficiently visible to establish both required connections.", "Oversized post caps and stepped bases dominate the smaller mechanical details and obscure upper mechanism relationships." ] }, "rationale": "A offers clearer hinge construction, functional bracing and more legible cable attachments throughout the sampled poses. B has a polished finish, but its raised deck and bulky upper supports conceal critical cable relationships. Continuous timing, loop smoothness, flicker and between-frame collisions remain unclear for both." } } ] }, "vanilla": { "score": 90, "rawScore": 90, "hardGate": false, "rawHardGate": false, "seconds": 321, "triangles": 26068, "images": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/hd/top.png" }, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "192 invalid vertices, degenerate faces, or zero-length edges" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "200.000m maximum extent; limit 4m" }, { "id": "finish_shading_profile", "passed": false, "earned": 0, "possible": 4, "detail": "0.0% smooth polygons for polished_smooth" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "agentVersion": "codex-cli 0.155.0-alpha.9.2", "evaluatorFingerprint": "07a858a97099d561a6161b825aaac0de8fc8e1cc84a73194cbb121bb97cee695", "generationTaskFingerprint": "1a47e1548eb2cabc9674cee1f96b10604b5692d3aa3d237101240d3298d40661", "rawScorerVersion": 5, "modelRequested": "gpt-6-astra", "sourceHashesRecordedAtGalleryBuild": true, "artifactHashes": { "asset.blend": "04cac220e4cd8071c814bf3a3fa49441b2099f7f7f4cffbd01de2f5c671762d8", "asset.glb": "224cabfdd8caa779cd9310766e3ceb5d45578a13c722d0af5d2f78f96dfed715" }, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [ "Studio ground" ], "sourceSha256": "04cac220e4cd8071c814bf3a3fa49441b2099f7f7f4cffbd01de2f5c671762d8", "sourceModified": false }, "imageHashes": { "perspective": "673374a523b7727561b2ef375246a75df7f8c4f241ee2a405649fb2424f77084", "front": "d5c864312777592630117b0a5f30b6a2c6acaad977bd1788e4f524e455554886", "back": "e163f09600362ac12507f9bd5d962d937b0247f1834421a083d454bdef66e812", "left": "a8b9a277c755a4925561f3577e33f4a226868e9a3c97db7b063c120f8ffa4090", "right": "4937297e0703576aa4198289bee31b9ce99c1272a76de638a93364f208124833", "top": "97e0f60970277b6b301af0d96307755a7459f3dcc32ce565abe98b26b4857130" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "2244ef92a819101fa3d67bf662a53001f9b2c16c9b85af6513fd12085ce522dc", "front": "011c91e234074544c35da33bd60e242642a08d9a940f4db4aebeedbc7c2cbec5", "back": "e400e5948080796ac2a773f986b36efc03b992361c6541c4fce198b2cde9bbaa", "left": "6f2e9347cd6aad249b19a65bc978057d87d65659b1c43493a878945b356df3a6", "right": "50edfe457b87ce3401ef1af510537b680c2280af21530334d656b033dd3ce93d", "top": "53269a0d4e71272a1ee62804800f695cde47f012a8c23bec7a70cc8430e2a731" } }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 685, "triangles": 22720, "images": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/hd/top.png" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fd73795a211744fff6462d9ed0b38296bb8b9c01e44ad7ca5a6f3a232b7deec7", "agentVersion": "codex-cli 0.155.0-alpha.9.2", "evaluatorFingerprint": "07a858a97099d561a6161b825aaac0de8fc8e1cc84a73194cbb121bb97cee695", "generationTaskFingerprint": "1a47e1548eb2cabc9674cee1f96b10604b5692d3aa3d237101240d3298d40661", "rawScorerVersion": 5, "modelRequested": "gpt-6-astra", "sourceHashesRecordedAtGalleryBuild": true, "artifactHashes": { "asset.blend": "d2d05f3125aeba202ae04f8deade7cb183f50bdb3d8a41cd82082597c377b789", "asset.glb": "7945c2898d79d5ebd996d9e90853c9b8c3c8f70afa0ac2544bb04de0f66d1bd1" }, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "d2d05f3125aeba202ae04f8deade7cb183f50bdb3d8a41cd82082597c377b789", "sourceModified": false }, "imageHashes": { "perspective": "09e89990d00a217e5739fd4971f70644674b98d4e6e5ca21340c83d3b81e38ef", "front": "d892339636331ec3562032600ed8708fa43b5c25eaf39fecfbf1e646bcc4c7a9", "back": "d87a6eb4f8fe864d314cd48c8a5759a5e72faa8c78d266ad44410a4013d5728e", "left": "37ac972f48b8009ec6553d06a02e39430a903f0ba2a628f6f83226b5127d5e9f", "right": "5c3d7567e2535e8dd70bfe9aae3f139c1dada7a8d491ee4df5b68a0f2a4614d3", "top": "8e3ea10a83c7f406951933d6e2e3149c84ecea68d46e7fbde50d8b35dd5b62e0" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6-astra--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "10f63adc061661aff291620ae42f78692c8ff39e2b75737e8870ac695d1a0ed3", "front": "a6edae6e6249a2746d8bd5c749af15ad4a0b3c401730763810413e47789073cf", "back": "5e7039f4e67503f0f3b0d839214458f73aa865674efb3745fa71a295f8f3748c", "left": "dc1909041944186b2f103d5f0e1c7bbc80ba4f5df62eeccbe3342fb831bdb1b4", "right": "c10409f5a5b9dad436172e435deeb872011cd4ae0cc9eda01878bd73c98c585b", "top": "8872b7f9836f7eca609c3989290841752c7c3853236a0e62f6f682904ec9479f" } }, "criteria": [ { "id": "bridge_parts", "label": "bridge parts", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "bridge_hinge", "label": "bridge hinge", "vanilla": { "pass": 2, "fail": 0, "unclear": 1 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "bridge_cables", "label": "bridge cables", "vanilla": { "pass": 0, "fail": 0, "unclear": 3 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } }, { "id": "bridge_finish", "label": "bridge finish", "vanilla": { "pass": 3, "fail": 0, "unclear": 0 }, "plugin": { "pass": 3, "fail": 0, "unclear": 0 } } ], "note": "The plugin result wins through clearer mechanical articulation, braced supports, richer functional detailing, and a raised deck that fits its frame. The no-plugin result has clean surface refinement but less convincing deck-to-frame proportions and weaker raised-state cable evidence. Neither sheet establishes continuous timing, loop smoothness, flicker absence, or between-frame collision clearance." }, { "task": "joinery_stool", "taskTitle": "Splayed-leg joinery stool", "model": "gpt-6.1-sol", "modelTitle": "GPT 6.1 Sol", "vanilla": { "score": 98, "rawScore": 98, "hardGate": false, "rawHardGate": false, "seconds": 519, "triangles": 13912, "images": { "perspective": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/hd/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/hd/bottom.png" }, "imageHashes": { "perspective": "1b4af9fec1d097d1a1f1283849fdcb2e1effa6f96c33d0d016a172adaa28df15", "front": "ef2ba3df18a1c23b61d8502f85bc9edeb8eb3281f0912e0a3635c1b60d33ca63", "back": "089553bc41c36b43d26078c920f3c309015707f0d98c4c984236811f654d7c68", "left": "4fe7933344881d84dd5e1d981be5e29e86ae9c8731a9d786054f74e139bd6dfd", "right": "8b3c6ee0b9b9a2ecdaa1c45f52b526785b39d1ad6767aa0d0cbbce191bec1a2f", "top": "2d7cfc54b4181a4ff6e35fc138e1360f239ec66d664804d47038409b9ade13ef", "bottom": "54fb3fa65c9387159b9466ab7c8097295a752a2d2f6fd9956c5bc38e5ac79201" }, "failedChecks": [ { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "200.000m maximum extent; limit 3m" } ], "artifactHashes": { "create_asset.py": "fd407dcd4e475ea6d8ed0594d29c14fe9f703e495826da74c1176353cf152d4b", "asset.blend": "688d00d1e1a11a7588c752618be2c8b7e913c84dacae623cfc7de59da3b146ee", "asset.glb": "1ee6f3a01c02ffc08596a51b40a805cc489d5264a7ef4e4903ab9a9747b7ec9e" }, "generationTaskFingerprint": "28c3c6ad61d6de6e939213c55613d617296ab05d5b926a06521a998c7ea82fe8", "correctedTaskFingerprint": "2cbfc0d0f305e997e246fcd1d18761664f42e38cc53f721e487550d43101381f", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "agentVersion": "codex-cli 0.160.0", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": null, "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "baseline", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [ "Studio | ground" ], "sourceSha256": "688d00d1e1a11a7588c752618be2c8b7e913c84dacae623cfc7de59da3b146ee", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/original/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/vanilla/original/bottom.png" }, "originalImageHashes": { "perspective": "8c80b1237f052326a8d7ec01cb586609681818ee78d7935a80d56575dbc0e814", "front": "11778435b41e9a8fee71fbcdc21dd0aa061e4a02446e47af87a9d5cf03250f65", "back": "e329a1425e3a721a7990f30ff43ac714d3d5d777d57594b58b4b97ed5e619de0", "left": "54702ede1963bfc9e5e8a00ed2ee72f45061ac4e6a8851e81eb467a8eab3f856", "right": "cd0dfea4667a647b0bd5ad098bc3400df1ba121b88a22d1341b4ab2af9ee3c73", "top": "7b7c60d5d1986da3a1c41060b53f031772bf392d00898e3c78094147b093df2a", "bottom": "c24e3ea2c77da1e3cf21bf5f4898846bb20a845da3be1c69d7b9b19241d34349" } }, "plugin": { "score": 98.33, "rawScore": 98.33, "hardGate": true, "rawHardGate": true, "seconds": 570, "triangles": 28570, "images": { "perspective": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/hd/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/hd/bottom.png" }, "imageHashes": { "perspective": "50bdd982b5e65dfbd7868877ed9406627b7b19f171ca23b49a63e79651e9468e", "front": "d10655ec9d525b52d896fffa0638af0851ec332ac9e7fa709f8f0345879d0fec", "back": "89558c5f0f7038f33f0cd6d11f582021ce9306b3bf5ad391035a46b2a62fead1", "left": "30c0d3caf6700ecf568886c3f3ba17416d7235e679a7193688488e438632e3cc", "right": "bf3f18319ddd645359c6a010701d5119d15c292f0aad0247a517a05cf47685c5", "top": "bdbaae20ef49bbd234c8f9b049a8f97788ca2e41635a6fc6e0f306e501cbc336", "bottom": "37f6d61602b50c2ace634b487ad80fb84a4504023e17b02a7fa00990f2691302" }, "failedChecks": [ { "id": "material_count", "passed": false, "earned": 3.333333333333333, "possible": 5, "detail": "2 materials; minimum 3" } ], "artifactHashes": { "create_asset.py": "96c3d9d32d38a40f1efc34cfb43dccdaf08bc07f4360b988834e11660e8ce139", "asset.blend": "76c463f0ee79b27ccf7c950f3a9ed5e96940b162a27745e3e6406949dbcc2779", "asset.glb": "7d6fb7c4f09ca92c6ddac9c978597edbd6276642e13ec6edab7d2bfe25a9ccfb" }, "generationTaskFingerprint": "28c3c6ad61d6de6e939213c55613d617296ab05d5b926a06521a998c7ea82fe8", "correctedTaskFingerprint": "2cbfc0d0f305e997e246fcd1d18761664f42e38cc53f721e487550d43101381f", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "agentVersion": "codex-cli 0.160.0", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "skills", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "76c463f0ee79b27ccf7c950f3a9ed5e96940b162a27745e3e6406949dbcc2779", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/original/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--gpt-6.1-sol--r01/plugin/original/bottom.png" }, "originalImageHashes": { "perspective": "6d85f114de915aa2f10ce725b173c815b9242616c684c0a89925392a1f051285", "front": "07b9104005b5e60433695681b9472d214968f0980df5a9b659ef2becd16a1636", "back": "e6498cfe277ffa3d25b584c65d5ee34b06a4892ff67292b82b155dccfd2e9fc7", "left": "fac3b5bb00e932d0001c54f0d0521f3b86697d863a59633095b5e9ede04cd824", "right": "5900cccd9e623d2699e8851c8b2b15a47a56cfa4b78dcf41a5bd584f3b1bf276", "top": "230be961ffe2f22803a064370c32158e59b61c246aa2ebf6dca5a0d9a6220460", "bottom": "d7fa33aaa5a23c6ade1d82c56cb2f176a120534325ffe11d4e5c6e167eb31815" } }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "note": "One review noted: The plugin result presents a complete, coherent wooden stool with splayed legs, a connected stretcher ring and visibly supported joints. The no-plugin result is only an unfinished slab. Absolute 0.55 m height cannot be verified from these images.", "criteria": [ { "id": "seat_support", "label": "seat support", "question": "Do all four leg tops meet the seat underside without open seams or piercing its upper surface?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "stretcher_joints", "label": "stretcher joints", "question": "Do all eight stretcher ends meet their legs without floating gaps or exposed protruding ends?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "ground_support", "label": "ground support", "question": "Do all four feet terminate on the same plane without visibly hovering or being buried?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "negative_space", "label": "negative space", "question": "Are the spaces under the seat and inside the stretcher ring intentionally open and free of accidental crossing geometry?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "assembly_count", "label": "assembly count", "question": "Are exactly four legs and four stretchers visible across the views?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "finish", "label": "finish", "question": "Are curved edges, wood finish and inset hardware coherent on reverse sides and underside as well as the hero view?", "critical": false, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "rawReview": { "taskId": "joinery_stool", "repetition": 1, "baselineAutomatedScore": 98, "candidateAutomatedScore": 98.33, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "current", "current" ], "criterionMajorities": [ { "criterionId": "seat_support", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "stretcher_joints", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "ground_support", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "negative_space", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "assembly_count", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "finish", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "seat_support", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "stretcher_joints", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "ground_support", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "negative_space", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "assembly_count", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "current" }, "decodedWinner": "current", "scoresByMode": { "vanilla": { "taskFidelity": 1, "silhouetteAndProportion": 1, "constructionPlausibility": 1, "craftsmanshipAndDetail": 1, "materialsAndReadability": 1, "surfaceFinishAndShading": 3, "materialTextureQuality": 1, "lightingAndPresentation": 5, "finalStageCompleteness": 1, "multiviewConsistency": 8 }, "current": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.99, "scores": { "A": { "taskFidelity": 1, "silhouetteAndProportion": 1, "constructionPlausibility": 1, "craftsmanshipAndDetail": 1, "materialsAndReadability": 1, "surfaceFinishAndShading": 3, "materialTextureQuality": 1, "lightingAndPresentation": 5, "finalStageCompleteness": 1, "multiviewConsistency": 8 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "seat_support", "A": "fail", "B": "pass", "evidenceA": "Only a thin slab is visible; no supporting legs exist.", "evidenceB": "Leg tops contact the seat underside in the perspective and underside views; the upper seat surface remains uninterrupted." }, { "criterionId": "stretcher_joints", "A": "fail", "B": "pass", "evidenceA": "No stretchers or leg joints are visible.", "evidenceB": "The perspective, reverse and underside views show the four rails terminating against the legs without visible gaps or exposed ends." }, { "criterionId": "ground_support", "A": "fail", "B": "pass", "evidenceA": "No feet are present.", "evidenceB": "The orthographic elevations show aligned foot bottoms contacting the floor without visible hovering or burial." }, { "criterionId": "negative_space", "A": "fail", "B": "pass", "evidenceA": "The required under-seat structure and stretcher ring are absent, so their intended openings are not formed.", "evidenceB": "Openings beneath the seat and within the stretcher ring are clear, with no accidental crossing members." }, { "criterionId": "assembly_count", "A": "fail", "B": "pass", "evidenceA": "Zero legs and zero stretchers are visible.", "evidenceB": "Across the perspective and underside views, exactly four legs and four perimeter stretchers are visible." }, { "criterionId": "finish", "A": "fail", "B": "pass", "evidenceA": "The plain white slab lacks rounded seat corners, wood finish and inset hardware.", "evidenceB": "Rounded seat edges, smooth wooden members and restrained recessed hardware remain coherent across opposing sides; the underside is finished." } ], "majorDefects": { "A": [ "Entire leg and stretcher assembly is missing.", "Seat remains a thin, sharp-cornered white slab.", "No visible wood material or joinery hardware." ], "B": [] }, "rationale": "B presents a complete, coherent wooden stool with splayed legs, a connected stretcher ring and visibly supported joints. A is only an unfinished slab. Absolute 0.55 m height cannot be verified from these images." } }, { "mapping": { "A": "current", "B": "vanilla" }, "decodedWinner": "current", "scoresByMode": { "current": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.5, "lightingAndPresentation": 8, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 1, "silhouetteAndProportion": 1, "constructionPlausibility": 1, "craftsmanshipAndDetail": 1, "materialsAndReadability": 1, "surfaceFinishAndShading": 3, "materialTextureQuality": 1, "lightingAndPresentation": 5, "finalStageCompleteness": 1, "multiviewConsistency": 8 } }, "result": { "winner": "A", "confidence": 0.99, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.5, "lightingAndPresentation": 8, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "B": { "taskFidelity": 1, "silhouetteAndProportion": 1, "constructionPlausibility": 1, "craftsmanshipAndDetail": 1, "materialsAndReadability": 1, "surfaceFinishAndShading": 3, "materialTextureQuality": 1, "lightingAndPresentation": 5, "finalStageCompleteness": 1, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "seat_support", "A": "pass", "B": "fail", "evidenceA": "Underside corner contacts and side views show four leg tops meeting the seat; the upper surface remains uninterrupted.", "evidenceB": "Only a thin slab is visible; no supporting legs meet its underside." }, { "criterionId": "stretcher_joints", "A": "pass", "B": "fail", "evidenceA": "The hero and underside views show a closed four-stretcher ring with ends meeting the legs, without visible gaps or projecting ends.", "evidenceB": "No stretchers or leg joints are visible." }, { "criterionId": "ground_support", "A": "pass", "B": "fail", "evidenceA": "Front, back and side views show aligned foot terminations contacting the floor without visible hovering or burial.", "evidenceB": "There are no feet or four-point ground support." }, { "criterionId": "negative_space", "A": "pass", "B": "fail", "evidenceA": "Openings beneath the seat and within the ring remain clear, with no accidental crossing members.", "evidenceB": "The required under-seat assembly and enclosing stretcher ring are absent, so the intended openings are not formed." }, { "criterionId": "assembly_count", "A": "pass", "B": "fail", "evidenceA": "The perspective and underside views establish exactly four legs and four perimeter stretchers.", "evidenceB": "Zero legs and zero stretchers are visible." }, { "criterionId": "finish", "A": "pass", "B": "fail", "evidenceA": "Rounded seat corners, softened edges, wood coloration and restrained recessed hardware remain coherent across opposing and underside views.", "evidenceB": "The slab has sharp square corners, a plain white material and no visible wood finish or inset hardware." } ], "majorDefects": { "A": [ "Wood grain is somewhat uniform and lacks convincing end-grain differentiation.", "The hero view crops the nearest foot, weakening presentation." ], "B": [ "Only an unfinished thin square slab is present.", "All four legs, all four stretchers and their joints are missing.", "Rounded seat geometry, wood material and inset hardware are absent." ] }, "rationale": "A presents a complete, visibly connected wooden stool with consistent splay, clear openings and refined edges. B remains a plain slab and fails the essential assembly requirements." } } ] }, "id": "joinery_stool--gpt-6.1-sol--r01", "repetition": 1, "cohort": "October 2026 spatial pilot", "limitation": "One generation per condition; two counterbalanced Astra judgments of the same images. Vanilla runs were collected later and paired with the existing frozen plugin runs; generation order was not counterbalanced. With plugin means pinned skill workflows and bundled scripts, not an MCP ablation. Concurrent times are not a latency benchmark. Structural scores are proxies, not visual quality or proof of contact.", "judgeCount": 2 }, { "task": "task_lamp_clearance_holdout", "taskTitle": "Articulated task-lamp clearance holdout", "model": "gpt-6.1-sol", "modelTitle": "GPT 6.1 Sol", "vanilla": { "score": 89.83, "rawScore": 89.83, "hardGate": false, "rawHardGate": false, "seconds": 479, "triangles": 35940, "images": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/hd/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/hd/bottom.png" }, "imageHashes": { "perspective": "f5bac35e2876a6dbeecd3cb7b829053c9c80375aa2bd4fefa067730a1f51490a", "front": "fd706d58c7daa24e4c20f0119435e1b914b1d331fd59e1d7e1690254bb3eb2e4", "back": "4b6edfc2ac5b1ea54dc9b0946cc4407d1fd76127f6b7452efe9775970f23e06c", "left": "1566dfe456ae1170227e9fd55deec019cff8df7f07240fb1f7a79e172d6d7552", "right": "e408f2e066777dcb562102340d3b276b20a1c5efca19c733095d41a216244f85", "top": "1b33154d412319d7c9d676dd1385657629301c612738828690a4e55bed13d1b4", "bottom": "a50360611bc852ca078c8a14a8ad288d1349cfe3728f66ec69fda0c7b8d721ab" }, "failedChecks": [ { "id": "semantic_part_coverage", "passed": false, "earned": 20.833333333333336, "possible": 25, "detail": "5/6 required semantic part groups found" }, { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "612 invalid vertices, degenerate faces, or zero-length edges" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "200.000m maximum extent; limit 3m" } ], "artifactHashes": { "create_asset.py": "d0d148e28cd654f263215a1794602d26e0058ad2dd1f61df42b8d4fbb8b43178", "asset.blend": "8e25a65fc783e501efa00a17758dccddbceb77677a68c14c681d0c7f9e774698", "asset.glb": "b3cd860021e4c6c7cf3e5da1490dccea8ec448941822ca0ea6cd8f8d38570613" }, "generationTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "correctedTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "agentVersion": "codex-cli 0.160.0", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": null, "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "baseline", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [ "Studio | desk plane" ], "sourceSha256": "8e25a65fc783e501efa00a17758dccddbceb77677a68c14c681d0c7f9e774698", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/original/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/vanilla/original/bottom.png" }, "originalImageHashes": { "perspective": "4c17067bb3dbcd7884e1607a042c96b994c6609ca5b9caa520cc0ef3bd190f45", "front": "62ad362dc9c71c3c5b045148081322e76a03d4015a0c41c06c72bd8ae9cdf483", "back": "d80e9527dc9d6488d3903553b2e4a0ce4fca14aed6a512bcb0d239c28a7d40e4", "left": "e2f566cf89f279376666c65051995efb5227a779887b112634b142492cbe4907", "right": "78f30e388ccffff657d3d728c889c2bc1f09ba8b533416ac0321821b38d73607", "top": "4662dbd1723aaa81d2b1a4c5903c5646b822d9fcbda3be3690137835acc32d70", "bottom": "444d0ac0de47bce27cf81b9cf1cb08383b38cd6420e73a19da5dc1a52bc4bf1c" } }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 672, "triangles": 32448, "images": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/hd/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/hd/bottom.png" }, "imageHashes": { "perspective": "7ee32846b0c13f71b1fc8cc658f43c40c346e07fa3c2ef411340ef1b7a06187a", "front": "f96627e13fbc403d98008dc13fe0c340062f4890e3b844fec6c94c9e3c3afc4e", "back": "8ad5db11119931579ec73114b7d003f97e184d877127fd16956d676025c312b7", "left": "120ba84f4772d913089ad07741883ec7bcc9527eb79121b8cea3ca1c7fd8d067", "right": "6f3b33b6034eac594c2a98b596fa372e39003c3295c0041376e5090001bce33a", "top": "149be728bd2c15fff9e535c590a8360e1844891b344588fab1cbe5bba72c434f", "bottom": "6e91321a29d4071564b9ec490caa1e6181941ac691cdb3735efd6365786f16f9" }, "failedChecks": [], "artifactHashes": { "create_asset.py": "80434358405cde7ae2ea8a05ea694c5ea0f769a95d84896042faf653d95e9c21", "asset.blend": "715466c1cc0c51a8d48a7cb6eac0282f6b5f298d2c8c615cfa0b7748000b6e68", "asset.glb": "4fb75fc3f5b2b5f3074efec091a8d92e081858e153ffea4605226ef30224b81a" }, "generationTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "correctedTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "agentVersion": "codex-cli 0.160.0", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "skills", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "715466c1cc0c51a8d48a7cb6eac0282f6b5f298d2c8c615cfa0b7748000b6e68", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/original/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--gpt-6.1-sol--r01/plugin/original/bottom.png" }, "originalImageHashes": { "perspective": "3a7efa0473f426a0ec46086784bea941cb13961087519115fc9bb7c31f5e55ed", "front": "f4cc7bf27fe56bcef3beb8bea9914fe31ac938cde82059bedb380af46b5e5534", "back": "2972ed024350c4b35a499601c37ce09e075e991c260f01ee824dd9d112d27380", "left": "ddb4b8e6690358a153d20ba88a852e47fa038e14dafd27b3924437023f2b69c0", "right": "35ff86e5a389761f56d075eb7c5882ffc2326e50092c76fa72b2b1ee938c336b", "top": "635521cffff05764ee575ba31940bbc2be25691746011cdd1ef10175dc3679e8", "bottom": "f16ff837c0a4b86dd9165ea4d7759a89f5012e2609545d9cf5f743c3d2ad5578" } }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "note": "One review noted: The plugin result presents a complete, smoothly finished lamp with visible hinge construction, open shade, recessed bulb, and grounded base and cable. The no-plugin result shows only a rectangular slab across its views.", "criteria": [ { "id": "joint_continuity", "label": "joint continuity", "question": "Are base-to-arm, elbow, and arm-to-shade connections physically continuous through credible hinge hardware?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "hinge_clearance", "label": "hinge clearance", "question": "Do adjacent arms and hinge cheeks retain readable mechanical clearance without unintended intersections?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "shade_cavity", "label": "shade cavity", "question": "Is the shade mouth open with a distinct recessed bulb that does not pierce the shell?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cable_contact", "label": "cable contact", "question": "Does the cable meet its socket and rest on the desk plane without floating or being buried?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "base_contact", "label": "base contact", "question": "Does the weighted base sit flat on the supporting plane?", "critical": true, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "finish", "label": "finish", "question": "Do enamel, brushed metal, rubber and reverse-side construction details read as a finished product?", "critical": false, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "rawReview": { "taskId": "task_lamp_clearance_holdout", "repetition": 1, "baselineAutomatedScore": 89.83, "candidateAutomatedScore": 100, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "current", "current" ], "criterionMajorities": [ { "criterionId": "joint_continuity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "hinge_clearance", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "shade_cavity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cable_contact", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "base_contact", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "finish", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "joint_continuity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "hinge_clearance", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "shade_cavity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cable_contact", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "base_contact", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "current", "B": "vanilla" }, "decodedWinner": "current", "scoresByMode": { "current": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 8, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 0, "silhouetteAndProportion": 1, "constructionPlausibility": 0, "craftsmanshipAndDetail": 0, "materialsAndReadability": 1, "surfaceFinishAndShading": 3, "materialTextureQuality": 1, "lightingAndPresentation": 4, "finalStageCompleteness": 0, "multiviewConsistency": 8 } }, "result": { "winner": "A", "confidence": 0.99, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 8, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "B": { "taskFidelity": 0, "silhouetteAndProportion": 1, "constructionPlausibility": 0, "craftsmanshipAndDetail": 0, "materialsAndReadability": 1, "surfaceFinishAndShading": 3, "materialTextureQuality": 1, "lightingAndPresentation": 4, "finalStageCompleteness": 0, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "joint_continuity", "A": "pass", "B": "fail", "evidenceA": "Visible brackets and pivot hardware connect the base, both arm segments, and shade.", "evidenceB": "Only a flat rectangular slab is visible; all lamp connections are absent." }, { "criterionId": "hinge_clearance", "A": "pass", "B": "fail", "evidenceA": "Front, back, and overhead views show spaced hinge supports and exposed pivot sections without evident unintended intersections.", "evidenceB": "No arms, hinge cheeks, or clearance relationships are present." }, { "criterionId": "shade_cavity", "A": "pass", "B": "fail", "evidenceA": "Front and underside views reveal an open shade, contrasting interior, and a distinct bulb recessed within the rim.", "evidenceB": "No shade cavity or bulb is visible." }, { "criterionId": "cable_contact", "A": "pass", "B": "fail", "evidenceA": "The cable visibly joins the base edge and lies along the supporting surface in the side views, ending in a plug.", "evidenceB": "No cable or socket is visible." }, { "criterionId": "base_contact", "A": "pass", "B": "fail", "evidenceA": "The round base has a flat lower pad aligned with the desk surface and a tight contact shadow.", "evidenceB": "The slab rests on the surface but does not provide the required weighted lamp base." }, { "criterionId": "finish", "A": "pass", "B": "fail", "evidenceA": "Smooth teal enamel, contrasting metallic hardware, dark cable, and an underside pad read as a finished product; brushed grain is subtle.", "evidenceB": "A plain pale slab lacks product materials, hardware, and reverse-side construction details." } ], "majorDefects": { "A": [], "B": [ "The entire articulated lamp assembly is missing.", "The visible asset is an unrefined rectangular slab with no requested functional details or material separation." ] }, "rationale": "A presents a complete, smoothly finished lamp with visible hinge construction, open shade, recessed bulb, and grounded base and cable. B shows only a rectangular slab across its views." } }, { "mapping": { "A": "vanilla", "B": "current" }, "decodedWinner": "current", "scoresByMode": { "vanilla": { "taskFidelity": 0, "silhouetteAndProportion": 1, "constructionPlausibility": 1, "craftsmanshipAndDetail": 1, "materialsAndReadability": 1, "surfaceFinishAndShading": 4, "materialTextureQuality": 1, "lightingAndPresentation": 5, "finalStageCompleteness": 0, "multiviewConsistency": 9 }, "current": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 8, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.99, "scores": { "A": { "taskFidelity": 0, "silhouetteAndProportion": 1, "constructionPlausibility": 1, "craftsmanshipAndDetail": 1, "materialsAndReadability": 1, "surfaceFinishAndShading": 4, "materialTextureQuality": 1, "lightingAndPresentation": 5, "finalStageCompleteness": 0, "multiviewConsistency": 9 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 8, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "joint_continuity", "A": "fail", "B": "pass", "evidenceA": "Only a rectangular slab is visible; all lamp connections are absent.", "evidenceB": "Visible pivot pins, brackets and hinge hardware connect the base, both arms and shade." }, { "criterionId": "hinge_clearance", "A": "fail", "B": "pass", "evidenceA": "No arms or hinge cheeks are present.", "evidenceB": "Front, rear and overhead views show separated hinge supports and spaced arm members around the pivot hardware." }, { "criterionId": "shade_cavity", "A": "fail", "B": "pass", "evidenceA": "No shade or bulb is visible.", "evidenceB": "Front and underside views reveal an open bell, an interior lining and a distinct recessed bulb without visible shell penetration." }, { "criterionId": "cable_contact", "A": "fail", "B": "pass", "evidenceA": "No cable or socket is visible.", "evidenceB": "The cable meets the base edge and lies along the supporting surface in the side views; the overhead view shows its continuous route to the plug." }, { "criterionId": "base_contact", "A": "fail", "B": "pass", "evidenceA": "The slab appears grounded, but the requested weighted lamp base is absent.", "evidenceB": "The round layered base has a level lower rim seated against the desk plane, with no visible floating gap." }, { "criterionId": "finish", "A": "fail", "B": "pass", "evidenceA": "A plain pale slab lacks product materials, hardware and reverse-side construction.", "evidenceB": "Smooth teal enamel, contrasting metal fasteners, dark cable and an inset underside pad read as a finished product; directional metal brushing is subtle." } ], "majorDefects": { "A": [ "Entire lamp assembly is missing; only a thin rectangular slab is shown.", "No articulation, shade cavity, bulb, cable or finished product detailing." ], "B": [] }, "rationale": "B visibly delivers the articulated lamp with credible hardware, readable hinge spacing, an open shade and recessed bulb, grounded cable and finished underside. A is a consistent but essentially empty slab blockout." } } ] }, "id": "task_lamp_clearance_holdout--gpt-6.1-sol--r01", "repetition": 1, "cohort": "October 2026 spatial pilot", "limitation": "One generation per condition; two counterbalanced Astra judgments of the same images. Vanilla runs were collected later and paired with the existing frozen plugin runs; generation order was not counterbalanced. With plugin means pinned skill workflows and bundled scripts, not an MCP ablation. Concurrent times are not a latency benchmark. Structural scores are proxies, not visual quality or proof of contact.", "judgeCount": 2 }, { "id": "tabletop_press--gpt-6.1-sol--r01", "task": "tabletop_press", "taskTitle": "Tabletop lever press", "model": "gpt-6.1-sol", "modelTitle": "GPT 6.1 Sol", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "tabletop_press", "repetition": 1, "baselineAutomatedScore": 91, "candidateAutomatedScore": 100, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "plugin", "plugin" ], "criterionMajorities": [ { "criterionId": "press_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_drive_chain", "baselinePasses": 1, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_supports", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "press_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_drive_chain", "baselinePasses": 1, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_supports", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "plugin", "scoresByMode": { "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.5, "lightingAndPresentation": 8.8, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 8.3, "silhouetteAndProportion": 7.6, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 7.2, "finalStageCompleteness": 8.1, "multiviewConsistency": 8.6 } }, "result": { "winner": "A", "confidence": 0.9, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.5, "lightingAndPresentation": 8.8, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.3, "silhouetteAndProportion": 7.6, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 7.2, "finalStageCompleteness": 8.1, "multiviewConsistency": 8.6 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "Base, C-frame, long lever, pivot, brass linkage, ram, circular die, exposed return spring, and rimmed tray are visible.", "evidenceB": "Base, upright frame, orange lever and pivot, toothed drive components, ram, die, upper spring assembly, and side tray are visible." }, { "criterionId": "press_drive_chain", "A": "pass", "B": "pass", "evidenceA": "Side views clearly connect the lever pivot and linkage to the vertical ram, centered over the die.", "evidenceB": "The lever hub sits beside the visible toothed drive, with the guided ram aligned over the die; the compact mechanism is less legible than A's." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "Spring is seated beneath the linkage, ram passes through the head, and tray rests on a visible bracket extending from the base.", "evidenceB": "Upper spring assembly has retaining plates and posts, ram has guide blocks, and the tray sits on an extended support." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded enamel housing, metallic tooling, brass linkage, fasteners, grip ribs, and inset plates create a convincingly finished industrial prop.", "evidenceB": "Smooth teal surfaces, contrasting orange lever, metallic tooling, fasteners, and rounded base members establish a finished manufactured appearance." } ], "majorDefects": { "A": [], "B": [ "Crowded drive components and short-looking lever weaken mechanical readability.", "Oversized presentation slab and smaller asset framing reduce useful detail visibility across the sheet." ] }, "rationale": "A wins through its clearer drive chain, stronger C-frame silhouette, better lever proportions, and more cohesive surface detailing. B meets the visible requirements, but its compressed mechanism and less effective framing make the finished assembly harder to assess." } }, { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 7.5, "silhouetteAndProportion": 7.2, "constructionPlausibility": 6.8, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.5, "lightingAndPresentation": 7, "finalStageCompleteness": 7.5, "multiviewConsistency": 8.5 }, "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.4, "lightingAndPresentation": 8.7, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.9, "scores": { "A": { "taskFidelity": 7.5, "silhouetteAndProportion": 7.2, "constructionPlausibility": 6.8, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.5, "lightingAndPresentation": 7, "finalStageCompleteness": 7.5, "multiviewConsistency": 8.5 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.7, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.4, "lightingAndPresentation": 8.7, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "Base, upright frame, orange lever, toothed pivot assembly, ram, circular die, upper spring, and side tray are visible.", "evidenceB": "Base, C-frame, long lever, pivot, brass linkage, rectangular ram, die, return spring, and collection tray are clearly visible." }, { "criterionId": "press_drive_chain", "A": "unclear", "B": "pass", "evidenceA": "The ram aligns over the die, but the connection between the offset toothed lever pivot and central ram is not clearly resolved in these views.", "evidenceB": "Side views clearly show the lever pivot and brass arm connecting to the spring-loaded plunger above the ram and die." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "Spring sits between upper hardware and housing; the tray rests on the base extension, and the ram is guided within the frame.", "evidenceB": "Spring has visible seats, pivot is carried by the frame, ram enters the head housing, and the tray is supported on the base." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Smooth teal coating, metallic tooling, contrasting lever and brass fittings, bevels, and fasteners establish a finished prop.", "evidenceB": "Controlled bevels, rounded frame transitions, inset plates, fasteners, grip ribs, and distinct painted, steel, and brass surfaces read as manufactured." } ], "majorDefects": { "A": [ "Lever-to-ram transmission remains visually ambiguous.", "Crowded upper mechanism and layered base weaken primary-form clarity.", "Small asset scale within the views limits inspection of functional connections." ], "B": [] }, "rationale": "B has a clearer mechanical chain, stronger C-frame silhouette, and more coherent manufactured detailing. Its larger, cleaner views make supported connections and surface refinement easier to verify. A appears finished, but its crowded drive assembly leaves a critical relationship uncertain." } } ] }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "vanilla": { "score": 91, "rawScore": 91, "hardGate": false, "rawHardGate": false, "seconds": 449, "triangles": 24008, "images": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "4d9afd4704c0bcb220bf4252378e1e4921dae50b444f5ed0e79aa1eb3e07f342", "front": "d2ae7959a4577bd0c42226fbe0742075905d2c6dc1bdfc925b62366931a22d32", "back": "a6dc65ad41480259e4fbd5611862e8ce14ee6f1a45f28f2c5b761ecb023b9f62", "left": "fb5dcb63bc5cfea1304e6de12b5afeba143f527375eb384e0d0163be1e192dcd", "right": "7160eae48ad10a0247b458b8cacf4f590bd1e2d4150d1f8be350b758959aae8f", "top": "85f4576d6855b38db47d948bd046ec92d5ebafa6185462bd28e0fdd029aabb97" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "482cfaf69fd8625fb5bfd67a9def4dd4322b888f1e755b13ddcc1cab848d0a78", "front": "83451ed525f3d958bd366d48d46c32c35fd733e49e03aa4ce717e0d25bce1943", "back": "69f05297f6250988ec27c741b81dd9d75b934fc1123043677bcdd5e584285071", "left": "b7c47c847d0a8c0385fa3e91f20ba2eb12b77cce61ee8564e4b1a7857dada14b", "right": "ee397b1be6173f176dac17522e45254028220c195c51f44e5f9b0e4f9a3276ab", "top": "936c5ea1cbe4ad709211171a54f2728cfe44c3a933f92612a51503bf345345a9" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "3143f0b1b526874bf99a0fc393b19addad8a6668b743fc5dd7cfac6f99079860", "front": "ee4be4b90fe82be0063e112250808de3233520c2dc04aeb43d0c7bf03b79f658", "back": "b2ec5e174fd6a8d8a3b25a329455a1744bf3f0e445f3da8081371b61a3dd669b", "left": "9e0131e6a5246e5c5bf4d4272f1936d17c60803eed6861923c370262e2fa06b2", "right": "4c992aaa3528d6cbb7f433a93dea4e384a00236071d18fc89e670de5409b253b", "top": "8b30bd6d7440f2283e90ad9b1a412085ec078c1aea61723a89858c30ccebc461" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "ca2bfd6c0a1e98d2ccabc01834756887ee5b36620db51b4b233b777d53dd596e", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "ca2bfd6c0a1e98d2ccabc01834756887ee5b36620db51b4b233b777d53dd596e" }, "artifactHashes": { "create_asset.py": "dc7d122a891af3a67a634e80d0c073f0f88054979a19262cb0ae3206662487db", "asset.blend": "ca2bfd6c0a1e98d2ccabc01834756887ee5b36620db51b4b233b777d53dd596e", "asset.glb": "752d19ca7f87132d74f78f4022e18f8bf1413c6c120a6e6f23010c34ffe1b0e6" }, "failedChecks": [ { "id": "triangle_range", "passed": false, "earned": 0, "possible": 5, "detail": "24008 triangles; expected 1800-24000" }, { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "14 invalid vertices, degenerate faces, or zero-length edges" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "e147da73ad0c4c84e79c127ae9eb1e37b1a299e26700ddfb96d17df5c0172260", "agentVersion": "codex-cli 0.160.0", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 5 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 516, "triangles": 22060, "images": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "27e24c2e0562e0521a60eb9b56b723becb1534c2e74c590cf6a196513ec68e89", "front": "f449512d8d8ae66443c5a741cfc08a7bf449c930f8a5c2e7808faf8f266a1d98", "back": "108b86beb47c7fc0cdbef8618b9c3dbcd2230dcfb2baa6f1c4a6b17a8ca9b87e", "left": "ad53e052d709217704c045fb40391c3946966c30bab09b30b75ed04368b214ab", "right": "7d392259696ef7215f406d12020a1ec161a377cefc7cfe6215b1933c51e1f1ef", "top": "0badf6ce0578cd166ed0f369ac3d2b2af6deddc7c56d7997b5404f56d7194668" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "9ed060e00529dfefbab2487db3226cbfac048bb6ce1c46becd4debba819bb34d", "front": "02beceb09653088c4c76b80c278f9e6567cfd5908898d1a375ef8ca5e9410c6b", "back": "28c6bf6a5adc4ddd125f471e72e04293ac0709f3057037aad577ad787f42785c", "left": "b422396a006dece03ccfa7cdbbd0ae4455dcdd1c5c846da8c52a153f0b3e3dc4", "right": "d97683e603eea8d4dcfba055b965d11313070dd89cd23d4d369e2c343fe16663", "top": "cbdb840f508f161bbf0fcd10911e14bfad0eda7ffca9b5a41c6b2b449fc795e3" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--gpt-6.1-sol--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "d91914c784943ee5dad8b143e565f71a0cc9f25802acfd9010664c22e766c0d3", "front": "3490cdcdce3f6cd2090273e1edf2f7179ea8ef2566436a6beb1c79f23b43e2a1", "back": "2344705b467fd8194ddc211a01ff5eb86be2aaf7d3688ec29a56532a072b006e", "left": "43016788bf1d129d8cbc562451295ed59d888f4024cecbf281590f7e86980021", "right": "de20455d30739ad15128f38ce366fe633f8b7c91ed5f2ae3c0d96d5aaec3cd01", "top": "3f72fbc8ae17ab7f0718577069c7dc59e400249894753971246ac1411f3fdee2" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "80ec6b4c2864cd5f555a019fa8768b317b1847178576ab228219100868aa7663", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "80ec6b4c2864cd5f555a019fa8768b317b1847178576ab228219100868aa7663" }, "artifactHashes": { "create_asset.py": "99c9d27d632b83d371da89c537d272a585367fa97a61598e3a62fd7e8c99777c", "asset.blend": "80ec6b4c2864cd5f555a019fa8768b317b1847178576ab228219100868aa7663", "asset.glb": "5d8b2c52952696cc9b6dfd966dc60b381a20646543f4c1eeb331b057a5d5c07d" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "e147da73ad0c4c84e79c127ae9eb1e37b1a299e26700ddfb96d17df5c0172260", "agentVersion": "codex-cli 0.160.0", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 5 }, "criteria": [ { "id": "press_parts", "label": "press parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_drive_chain", "label": "press drive chain", "vanilla": { "pass": 1, "fail": 0, "unclear": 1 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_supports", "label": "press supports", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_finish", "label": "press finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The plugin result wins through its clearer drive chain, stronger C-frame silhouette, better lever proportions, and more cohesive surface detailing. The no-plugin result meets the visible requirements, but its compressed mechanism and less effective framing make the finished assembly harder to assess." }, { "id": "winch_drawbridge--gpt-6.1-sol--r01", "task": "winch_drawbridge", "taskTitle": "Animated miniature winch drawbridge", "model": "gpt-6.1-sol", "modelTitle": "GPT 6.1 Sol", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. This gallery shows still views; reviewers also received sampled animation frames when available. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "winch_drawbridge", "repetition": 1, "baselineAutomatedScore": 98, "candidateAutomatedScore": 100, "hardGates": { "baseline": false, "candidate": true }, "visualWinnerVotes": [ "vanilla", "vanilla" ], "criterionMajorities": [ { "criterionId": "bridge_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_hinge", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_cables", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "bridge_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "bridge_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_hinge", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_cables", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.5, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 }, "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 9, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 8.5 } }, "result": { "winner": "B", "confidence": 0.85, "scores": { "A": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.5, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 9, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 8.5 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two masonry supports, planked deck, crossbeam, upper drum, side crank, hinge hardware and paired cables are visible across the views.", "evidenceB": "All required parts are clearly visible, including an exposed wound drum, crank mechanism, paired cable guides and deck anchors." }, { "criterionId": "bridge_hinge", "A": "pass", "B": "pass", "evidenceA": "The three poses show the deck rising around its base at the visible hinge hardware, with the supports stationary.", "evidenceB": "The lowered, intermediate and raised deck poses retain a consistent pivot at the visible basal hinge." }, { "criterionId": "bridge_cables", "A": "unclear", "B": "pass", "evidenceA": "The lowered views establish two anchored cables, but the beam, supports and raised deck obscure their upper routing and raised-state connections.", "evidenceB": "Paired cables lead through upper guides to deck-corner anchors; their visible spans shorten plausibly as the deck rises, with short connections remaining at the raised corners." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded masonry, warm finished timber, metal straps and repeated fasteners remain coherent across views and poses, without obvious blockout residue.", "evidenceB": "Consistent bevels, smooth shading, differentiated materials, deck fasteners and refined mechanical hardware sustain a finished miniature appearance." } ], "majorDefects": { "A": [ "The crossbeam and tall masonry supports substantially obscure the winch routing and raised-state cable connections." ], "B": [] }, "rationale": "B wins through clearer mechanical construction: the exposed drum, cable guides, crank assembly and hinge make the lifting relationship easier to verify. A is polished but conceals critical connections. Neither image sequence establishes continuous timing, loop smoothness, flicker absence or between-frame collision freedom." } }, { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 9, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.2, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "plugin": { "taskFidelity": 8.3, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.6, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.3, "lightingAndPresentation": 8.1, "finalStageCompleteness": 8.6, "multiviewConsistency": 8.6 } }, "result": { "winner": "A", "confidence": 0.84, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 9, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 9, "materialsAndReadability": 9, "surfaceFinishAndShading": 9, "materialTextureQuality": 8.2, "lightingAndPresentation": 9, "finalStageCompleteness": 9, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.3, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.6, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.3, "lightingAndPresentation": 8.1, "finalStageCompleteness": 8.6, "multiviewConsistency": 8.6 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two masonry-based supports, plank deck, crossbeam, wound drum, side crank, and two lifting cables are clearly visible.", "evidenceB": "All required parts are visible across the views; the rear view establishes the drum and paired cable windings." }, { "criterionId": "bridge_hinge", "A": "pass", "B": "pass", "evidenceA": "Visible hinge hardware stays at the deck root while the deck changes from lowered through intermediate to nearly vertical.", "evidenceB": "The deck's three poses are consistent with rotation about the fixed lower hinge hardware visible in the static views." }, { "criterionId": "bridge_cables", "A": "pass", "B": "unclear", "evidenceA": "Paired cables run from upper guides to deck-corner fittings, shortening plausibly as those fittings approach the guides in the raised pose.", "evidenceB": "Both cables and deck attachments are established when lowered, but the large pillars and crossbeam obscure enough of the intermediate and raised connections that both complete paths cannot be verified." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Consistent bevels, smooth drum windings, plank fasteners, metal fittings, and restrained material separation produce a finished miniature.", "evidenceB": "Rounded masonry, finished timber, repeated bolted straps, and smooth metal fittings remain coherent across the supplied views and poses." } ], "majorDefects": { "A": [], "B": [ "Animation framing conceals much of the winch and cable connections behind the tall supports, limiting verification of the critical lifting relationship." ] }, "rationale": "A wins through clearer mechanical presentation, balanced proportions, and more legible cable routing throughout the supplied poses. B is polished and structurally convincing, but its bulky supports obscure the mechanism. Continuous timing, flicker, and between-frame collisions remain unverified for both." } } ] }, "votes": { "baseline": 2, "candidate": 0, "tie": 0 }, "vanilla": { "score": 98, "rawScore": 98, "hardGate": false, "rawHardGate": false, "seconds": 418, "triangles": 18946, "images": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "b1f4d74dd9c1daa095fff803f3824c3f17ebc76e7c797236419f10db9a5b86ff", "front": "a46bffe75416c9757f41d651bf93ecd1da4696a5071813527e3b0e36974d7351", "back": "86d1b5777abf85fff6718ebf0b362ae4308f15fdaa67e87666869e5928f82ec2", "left": "f3a376f9fd6c0dba397f70a807b3fdab2254ecf6304313b06c67cdb735eb02ad", "right": "019c322a1f3c97427c2615534e80b66263a540741e5c0a9f8cbce0cd4d83439f", "top": "ab1f4ad7d5ac343c6fe3c0b7b762feee2888c8713467e956be2493e1b17b215a" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "0a3f81b11ec7728fe53509ffb598bfef9d0c64785dbeee5bf913c2dcc7a2e404", "front": "11b4db1af8d914ef13a4b60e33398c3721a07bcb6dd2353bcf5a4f23f7b95822", "back": "d747dd8df7e41bc97be03e507695b64525f3fe3f689be3f602bb016dc8aedc49", "left": "068173ae4212cbddd8194791b405ae1155b8adffd46dc44e726c50e6843f96e1", "right": "39b77134470d3df53f8964448476ff2b1f74f818f3f5125444d165cac6ecd9ae", "top": "ad0658be430a716b8a2946890708e46a43519e525704d030d1f6fbd44c038b01" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "a56fe54dfef06f9092552e3981ff573ee1ba80aac56edf916c3c9afdc488026d", "front": "e3111a97bc5bfebdde79834105ff60ca5d77789f08f4e5b290cf7dbf4f7d5647", "back": "2dda0a623732eabeeb856370d1e95554b47730daafdec6b57e6ef82dfb6291ca", "left": "dff499c5743f7bb2b88812d1b87947ef52ef666e41bbac0024fa768eedc8dd6b", "right": "7d97cde26b9195a7f4d1c44c83ba700cf9a1a7a3d66ef1a6b1e41145416305ee", "top": "0871fe6f6e50c5e00bf5124fb2c2adfaa8157342a341cbd2770f03272b686fe5" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "f4070a446f79cf3a63a5764d385c7df3ebc3fbe8d08d85e434a2051b89a3e8df", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [ "Studio shadow floor" ], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [ "Studio shadow floor" ], "sourceSha256": "f4070a446f79cf3a63a5764d385c7df3ebc3fbe8d08d85e434a2051b89a3e8df" }, "artifactHashes": { "create_asset.py": "465b2b876f12ac279716d12dfcf7225067f29e0200c68d8be039af53d1fcd78f", "asset.blend": "f4070a446f79cf3a63a5764d385c7df3ebc3fbe8d08d85e434a2051b89a3e8df", "asset.glb": "c2b45e7cdb8ab4b2788d79dab201428f225cabb96f24791d571f6d126f539cc2" }, "failedChecks": [ { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "200.000m maximum extent; limit 4m" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "1a47e1548eb2cabc9674cee1f96b10604b5692d3aa3d237101240d3298d40661", "agentVersion": "codex-cli 0.160.0", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 5 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 691, "triangles": 24200, "images": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "220114ae3a068c4cee78170cbe139fcff1a8f6d45d8eb3234284ac77ca2aa25c", "front": "625a373e609c1521e5637bd943f709a0fd59fb9c4574676034621a0261d8974f", "back": "d62a2b901e876791f922d70ed8ee7dc5337009c77b3041563657161dd136a2a0", "left": "5b7517274b5b1796c9051b2c834cd7aa2f033aeff913e6d2c9bda3bb236be491", "right": "8d59d063aaeaa1157242f378ebb7df116dcb18605a6225e16c63864a6354f87f", "top": "fd82d5c31d7f8005ead11e5644f23184cfc03605e9dbf1b9b5b1ed5814bb7fc0" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "83b5ffbda4761909725563c55243c911f4a0e2e5354eb27a667a6ab7b66aa5f0", "front": "52e326ac5580f3aa673e572be0e073ab565c0e77860da3a59db71494f274c8f6", "back": "f60d4211e24ca393c8f9ec3371671a09da5f5b7030982ec7864ac5c690b18254", "left": "d6e4e8b3ac71cd69be81ad1630108d465e5ea9540f9ba6cb524c2fc85465e1ea", "right": "90564d8538e2fc44aa64ea90639556d5602712c47dd0460fba88ec260b7628dc", "top": "9a6939f2647c88debe782609be665816daf819c6b1139b95a15dbd6a5a7b6a36" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--gpt-6.1-sol--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "1076dd373c74c619be8c777b48b30812761ca5b74eb77918b7229b1cbf780f66", "front": "4e6d41ac0885acde52f2c642d526c0fa8bda10e9910c74167f111b7dd3eddc01", "back": "ef87a58939dd227d9eddfbafaf803265cc1a9d616a1104d4ff9d6ac38024f5cd", "left": "1351afee8bf5064f1859deaadedc91b2c2929abe21e7fd9b0efeb1dece157843", "right": "8558d9f193436aa0cb1c6745d7d7498bcb7e8be609ca2b0b3a78b5421dbcde5a", "top": "6799d3b17b356f4b5da16f0accf9b2ba129753f775ff72bd51d1b9662b31dad6" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "755fece70febb11cda643c5cbc9b0ed532836acadd46de17cd2e9a3b7c987768", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "755fece70febb11cda643c5cbc9b0ed532836acadd46de17cd2e9a3b7c987768" }, "artifactHashes": { "create_asset.py": "ad33ab770b2960be718087944db15f305cfe5c7742f911c22c9f78a420ef9223", "asset.blend": "755fece70febb11cda643c5cbc9b0ed532836acadd46de17cd2e9a3b7c987768", "asset.glb": "25c9482670dad74d72de11e2a28aa3161c5fff9af8ebb3f787bdb3298722c638" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "1a47e1548eb2cabc9674cee1f96b10604b5692d3aa3d237101240d3298d40661", "agentVersion": "codex-cli 0.160.0", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 5 }, "criteria": [ { "id": "bridge_parts", "label": "bridge parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "bridge_hinge", "label": "bridge hinge", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "bridge_cables", "label": "bridge cables", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 0, "unclear": 2 } }, { "id": "bridge_finish", "label": "bridge finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The no-plugin result wins through clearer mechanical construction: the exposed drum, cable guides, crank assembly and hinge make the lifting relationship easier to verify. The plugin result is polished but conceals critical connections. Neither image sequence establishes continuous timing, loop smoothness, flicker absence or between-frame collision freedom." }, { "task": "joinery_stool", "taskTitle": "Splayed-leg joinery stool", "model": "claude-opus-5-5", "modelTitle": "Opus 5.5", "vanilla": { "score": 96, "rawScore": 96, "hardGate": true, "rawHardGate": true, "seconds": 149, "triangles": 4708, "images": { "perspective": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/hd/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/hd/bottom.png" }, "imageHashes": { "perspective": "7f7819e6073f59a8830a82782563207bc22adf72abbdfb02701e700e9c953a68", "front": "27839b003b2e40ec8734541236248051f6251aa1e5144c621e8b7ef765cd5c38", "back": "960cc41fd776d0eb350f9b9c0ed0f501ecd29c0e911320a0cb324da2b53518db", "left": "f48ff823cf01be22c78dcc4d472dae1877c3e6b43ec1115710e235a6691e48cd", "right": "05068b70a42a1751b70941573016c73a59d6cf29114f2b0cbff5122bc98a6b7f", "top": "90097af7544a649d23ff13a09f5d6de7ed354093a6dbff6a6f1af366de67be89", "bottom": "6a762e624054ec0638579ba696b98e9b920fc8e780682a83324316610da079b9" }, "failedChecks": [ { "id": "uv_coverage", "passed": false, "earned": 0, "possible": 4, "detail": "0/43 meshes have UV layers; minimum ratio 0.80" } ], "artifactHashes": { "create_asset.py": "691bf52d13328bd14ce2a62ea5777195837f889b2d247cec708e5083cbad1418", "asset.blend": "38b483be81931fa33a327969b9484efc678e18580951dc5674a0f75eeaa3a9f2", "asset.glb": "80aac4d3e13c777281c55ebd2a8c58138a1776efb9a68938632904c5f957bfc8" }, "generationTaskFingerprint": "28c3c6ad61d6de6e939213c55613d617296ab05d5b926a06521a998c7ea82fe8", "correctedTaskFingerprint": "2cbfc0d0f305e997e246fcd1d18761664f42e38cc53f721e487550d43101381f", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "agentVersion": "2.1.287 (Claude Code)", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": null, "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "baseline", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "38b483be81931fa33a327969b9484efc678e18580951dc5674a0f75eeaa3a9f2", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/original/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/vanilla/original/bottom.png" }, "originalImageHashes": { "perspective": "9b2d50f808659f85ac55115a9f45db129cb72b79c525a53f1315e10bace7a574", "front": "90192a32462fec7bec12091a6c0ee988987613575e2146d39fce288318c405af", "back": "a0aabbedc95bbb7f8a84695c79bece32529ed99119ea9cdfb7a87ef45645fb6a", "left": "120adad6b5b77db9f10ca04fac129b33d1d4dbe7448277ce34c3a74fa045b290", "right": "6c5b223aa420e217f9ec908f74494cc1ce8f23b4a79027aa91b99f7e8c6ff923", "top": "4de363050c34da95c28c00388b1d4bcf03e6d0713a4057c5285969d61a9f9056", "bottom": "e5285a070ae4d169915f4789da39f2e1cb22a101b79c85841aa86a572fb0c2d2" } }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 206, "triangles": 6940, "images": { "perspective": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/hd/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/hd/bottom.png" }, "imageHashes": { "perspective": "2d56f52c59f5bed75539b6917d4338e92abf46761e31f11ed4057f6d2df81f52", "front": "6ee0fb63c026993d4695e54062776e1b9a8f96975512f4fa50d2283d31a94adf", "back": "ed7e3eb035d07bbb30f0901bc1a2f1e4b9575d720b90284b7ddc997990bc6a7f", "left": "05dac35b4d7126a67a08df41bb3a484e51078ab9b55721f9ccd5a97c9809e521", "right": "91d847d4a426b8b1bf7952c6296a626b2143b4b53edf5c68997ac3f04908e91f", "top": "24de6a3fbbcd485bac9d773aa4f62b80830c00db2e218523f1eafbb19900a775", "bottom": "727f4988e447a5114730634d39b3414a5ad8a8cf329d4556d4643146936623a3" }, "failedChecks": [], "artifactHashes": { "create_asset.py": "071ebd34c98cc30478e447a848f66ae5176fb8724a1714ac6f1029f33da80fcd", "asset.blend": "f650cd601fe633541d33c1357fbc35ab946fb2ea5b86d548c06653ad4b520af2", "asset.glb": "beb3ec3eaae5cbcdd828a301e20a8f8b11a5697d3ff848c2eb0bb01432abdf67" }, "generationTaskFingerprint": "28c3c6ad61d6de6e939213c55613d617296ab05d5b926a06521a998c7ea82fe8", "correctedTaskFingerprint": "2cbfc0d0f305e997e246fcd1d18761664f42e38cc53f721e487550d43101381f", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "agentVersion": "2.1.287 (Claude Code)", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "skills", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "f650cd601fe633541d33c1357fbc35ab946fb2ea5b86d548c06653ad4b520af2", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/original/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--claude-opus-5-5--r01/plugin/original/bottom.png" }, "originalImageHashes": { "perspective": "9a4625f9f76a6070d5150ef0d4e840fcba719a545d263824ef00ec817371efee", "front": "531167ec51574cefc646525724f6f7c481a347d4c761cc3c7b0f1ff8a954af70", "back": "c4a45f00c6b06e8786695a87ee273dca98c96d9975b97a2d65a534efd06f4e39", "left": "c9fc9ff067b568b7cefc30fd99873aaf9d6a29990a7efbe0abbfe0b4d4fb18d4", "right": "db5fe0d1e94117cc939db17783e7fe1dff2825df8eeeee56e94ea9bc1c39e1df", "top": "d53ec9698e273508334c295a770db356e2281f88353b3aaa6799ac5a6edd5a48", "bottom": "c20fab438ca56e1ded31251b37e3e0fb7880608f0f18fed11f63d35f99721b0c" } }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "note": "One review noted: The plugin result wins through clearer inset hardware, more restrained material treatment and consistently resolved timber joints. Both visibly satisfy the structural relationships and preserve open space. Both have limited readable wood grain, but the plugin result presents the more coherent finished stool.", "criteria": [ { "id": "seat_support", "label": "seat support", "question": "Do all four leg tops meet the seat underside without open seams or piercing its upper surface?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "stretcher_joints", "label": "stretcher joints", "question": "Do all eight stretcher ends meet their legs without floating gaps or exposed protruding ends?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "ground_support", "label": "ground support", "question": "Do all four feet terminate on the same plane without visibly hovering or being buried?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "negative_space", "label": "negative space", "question": "Are the spaces under the seat and inside the stretcher ring intentionally open and free of accidental crossing geometry?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "assembly_count", "label": "assembly count", "question": "Are exactly four legs and four stretchers visible across the views?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "finish", "label": "finish", "question": "Are curved edges, wood finish and inset hardware coherent on reverse sides and underside as well as the hero view?", "critical": false, "vanilla": { "pass": 0, "fail": 1, "unclear": 1 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "rawReview": { "taskId": "joinery_stool", "repetition": 1, "baselineAutomatedScore": 96, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "current", "current" ], "criterionMajorities": [ { "criterionId": "seat_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "stretcher_joints", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "ground_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "negative_space", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "assembly_count", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "finish", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "seat_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "stretcher_joints", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "ground_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "negative_space", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "assembly_count", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "current", "B": "vanilla" }, "decodedWinner": "current", "scoresByMode": { "current": { "taskFidelity": 8.7, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.4, "craftsmanshipAndDetail": 8.2, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 6.7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.3, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 7.7, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 7.8, "materialTextureQuality": 6.7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 7.9, "multiviewConsistency": 8.8 } }, "result": { "winner": "A", "confidence": 0.8, "scores": { "A": { "taskFidelity": 8.7, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.4, "craftsmanshipAndDetail": 8.2, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 6.7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.3, "multiviewConsistency": 9 }, "B": { "taskFidelity": 7.7, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 7.8, "materialTextureQuality": 6.7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 7.9, "multiviewConsistency": 8.8 } }, "criterionResults": [ { "criterionId": "seat_support", "A": "pass", "B": "pass", "evidenceA": "Four rectangular leg tops contact the seat underside; no visible seam or upper-surface penetration.", "evidenceB": "Four rounded leg tops meet the underside in the reverse view, without visible gaps or penetration." }, { "criterionId": "stretcher_joints", "A": "pass", "B": "pass", "evidenceA": "Perspective and underside views show the four rails terminating against the legs without exposed ends or floating gaps.", "evidenceB": "The rounded stretchers connect into the legs throughout the ring; no exposed protruding ends or gaps are visible." }, { "criterionId": "ground_support", "A": "pass", "B": "pass", "evidenceA": "Foot terminations align across the elevations and visibly contact the floor.", "evidenceB": "All four capped feet terminate consistently, with floor contact supported by the perspective and elevations." }, { "criterionId": "negative_space", "A": "pass", "B": "pass", "evidenceA": "The space below the seat and inside the stretcher ring remains open, without crossing geometry.", "evidenceB": "The ring interior remains open; the two underside battens stay against the seat rather than crossing the leg space." }, { "criterionId": "assembly_count", "A": "pass", "B": "pass", "evidenceA": "Exactly four legs and four perimeter stretchers are visible across the views.", "evidenceB": "Exactly four legs and four perimeter stretchers are visible; the additional underside pieces are seat battens." }, { "criterionId": "finish", "A": "pass", "B": "fail", "evidenceA": "Rounded seat corners, softened timber edges and small recessed-looking fasteners remain coherent across the views.", "evidenceB": "Seat fasteners read as raised wooden buttons rather than inset hardware. Strong dark transitions above the reflective foot caps weaken the otherwise smooth wood finish." } ], "majorDefects": { "A": [], "B": [ "Seat hardware appears proud rather than inset.", "Dark lower-leg shading and bright foot caps distract from the restrained wooden construction." ] }, "rationale": "A wins through clearer inset hardware, more restrained material treatment and consistently resolved timber joints. Both visibly satisfy the structural relationships and preserve open space. Both have limited readable wood grain, but A presents the more coherent finished stool." } }, { "mapping": { "A": "vanilla", "B": "current" }, "decodedWinner": "current", "scoresByMode": { "vanilla": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 7, "surfaceFinishAndShading": 8, "materialTextureQuality": 6, "lightingAndPresentation": 8, "finalStageCompleteness": 7.5, "multiviewConsistency": 8.5 }, "current": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 8, "materialTextureQuality": 6, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8.5 } }, "result": { "winner": "B", "confidence": 0.7, "scores": { "A": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 7, "surfaceFinishAndShading": 8, "materialTextureQuality": 6, "lightingAndPresentation": 8, "finalStageCompleteness": 7.5, "multiviewConsistency": 8.5 }, "B": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 8, "materialTextureQuality": 6, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8.5 } }, "criterionResults": [ { "criterionId": "seat_support", "A": "pass", "B": "pass", "evidenceA": "Underside shows four rounded leg tops contacting the seat; no visible open seams or upper-surface penetration.", "evidenceB": "Four leg tops terminate against the underside at the corners, without visible gaps or penetration." }, { "criterionId": "stretcher_joints", "A": "pass", "B": "pass", "evidenceA": "Perspective and underside views show the four rounded stretchers meeting the legs without exposed free ends.", "evidenceB": "Four rectangular stretchers form a connected ring, with ends terminating against leg surfaces." }, { "criterionId": "ground_support", "A": "pass", "B": "pass", "evidenceA": "Capped feet share a consistent baseline in the elevation views; perspective contacts show no evident hovering.", "evidenceB": "Flat leg ends align with the floor across elevations and the perspective view." }, { "criterionId": "negative_space", "A": "pass", "B": "pass", "evidenceA": "The ring interior and space beneath the seat remain open; underside battens stay against the seat.", "evidenceB": "The under-seat volume and stretcher-ring interior are open, with no accidental crossing members." }, { "criterionId": "assembly_count", "A": "pass", "B": "pass", "evidenceA": "Four legs and four perimeter stretchers are visible across perspective and underside views.", "evidenceB": "Four legs and four perimeter stretchers are consistently visible across the views." }, { "criterionId": "finish", "A": "unclear", "B": "pass", "evidenceA": "Rounded surfaces and underside finishing are coherent, but the pale circular seat details do not clearly establish inset hardware.", "evidenceB": "Softened seat edges, finished reverse surfaces and small recessed-looking fasteners remain consistent across views." } ], "majorDefects": { "A": [ "Inset seat hardware is not clearly readable.", "Wood surfaces have little visible grain, reducing material specificity." ], "B": [ "Wood surfaces remain largely uniform, with limited grain or fine material detail." ] }, "rationale": "B narrowly wins through clearer restrained hardware and more legible stretcher-to-leg construction. A has pleasing rounded refinement, but its seat hardware is ambiguous. Both show connected assemblies, open negative space and consistent ground contact; absolute height cannot be verified from these images." } } ] }, "id": "joinery_stool--claude-opus-5-5--r01", "repetition": 1, "cohort": "October 2026 spatial pilot", "limitation": "One generation per condition; two counterbalanced Astra judgments of the same images. Vanilla runs were collected later and paired with the existing frozen plugin runs; generation order was not counterbalanced. With plugin means pinned skill workflows and bundled scripts, not an MCP ablation. Concurrent times are not a latency benchmark. Structural scores are proxies, not visual quality or proof of contact.", "judgeCount": 2 }, { "task": "task_lamp_clearance_holdout", "taskTitle": "Articulated task-lamp clearance holdout", "model": "claude-opus-5-5", "modelTitle": "Opus 5.5", "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 198, "triangles": 23396, "images": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/hd/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/hd/bottom.png" }, "imageHashes": { "perspective": "9bdab66614c20538927d1692cdb29be55057f2cb5a82357cbb03accbb659e4ba", "front": "cd82322eb9210541082de304bfc48601e82150b411660ea2c4744f81aad95909", "back": "ec8464902cf44a3831e4dc2b9be17548be7e62f1441ca61056dc8e40c8cf1ec0", "left": "5650779eea45255b0f467db3a9863c7abb8335a86d3827c879c6b1d2663a5908", "right": "adba01c0381116748961c5f6da33dacf6df355bd3d5c0322dbc5c537834af13a", "top": "62d3770f5bc3134c280eb32b70b0e262779cb2604c666bc6965e5265385efd90", "bottom": "feb4872ba48c1da7d89f0a2080f839e28dbabb3e85c7731ecdc8e1975164f0f0" }, "failedChecks": [], "artifactHashes": { "create_asset.py": "da971b5b90a3b85e499a4742fdb15320a0e44ad4c71b8e4f2e2ecd75b05b768a", "asset.blend": "ed96c4180bebc4d0ef000c06081396372383271e8927351700ba08465cc0f65f", "asset.glb": "5550a00c31e9a32b6118ff61cc044581bbcbfe22055e4563746a114024c0fca7" }, "generationTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "correctedTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "agentVersion": "2.1.287 (Claude Code)", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": null, "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "baseline", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "ed96c4180bebc4d0ef000c06081396372383271e8927351700ba08465cc0f65f", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/original/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/vanilla/original/bottom.png" }, "originalImageHashes": { "perspective": "8bd11ce72bb9460ebaf0a268068a2431bc86dbbe307d2549fdd0739890e596d4", "front": "a7b491a1f816d3ee3bc9d5cb934457747a0f97296671b0c3a84e97684f95c4a0", "back": "611253105f31460fbd070da40d79b2a6fc50baee1933969efe373313746ada4c", "left": "a235fe5f3a6ffacb240446bfd306f0010361d1720c8c63bf41c18367e582df83", "right": "a1807ffdc19979a6a7613a7b758fd33a594a2a85b5876671e901558e7d40e5fd", "top": "59d23eeeb3583ac17ae6d9aa48c9f73420d6f93908df302e04be6bd584a225ef", "bottom": "5aa1d11567666ecae6c1049289cdb6acbe44f8bab2244adc0d3658126294040a" } }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 312, "triangles": 29556, "images": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/hd/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/hd/bottom.png" }, "imageHashes": { "perspective": "62a4f335b258047ee2d15e66c65a3c9f8b05d4e3278e20512ee1c5865ab653ed", "front": "44795a547fe8e426e25419239d392f8ddbdbb73d48bb7fc2b9f2b9d0c558577b", "back": "b4fba7ca090ee1f715bf30ca1c924696ea8eb229f5fed986739a397684f55572", "left": "11b7d57db9cffd1b0b4d6ca57b557c4ad7f2f5f909f6b0f5bd7d84fe286d28db", "right": "0a73eaf5702a467b0764367ba6eea3004a635f054ee3bb9bf0ee3cfbd1dbaf9e", "top": "ec0729ce87468799f270aaa2d38174f7a262d4c38d59ac7ceb021f2c940256f7", "bottom": "cf1a431926af9629d5803c6a434b1d785c794af8673e6cbcfa60b5cec15ea2a1" }, "failedChecks": [], "artifactHashes": { "create_asset.py": "cb1314aa1117d056bdf7f7a6c492046f238e5aa87b09919dcc1efba50b874626", "asset.blend": "eed5f9e367b4c67651ed7b96425dbc048359500a971c5acf0ab0fcb7548f465b", "asset.glb": "b56f040ba210ce056f41c645cbdb6f7fa74fd28d89c50f0a1957dc729d0ffcf1" }, "generationTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "correctedTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "agentVersion": "2.1.287 (Claude Code)", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "skills", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "eed5f9e367b4c67651ed7b96425dbc048359500a971c5acf0ab0fcb7548f465b", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/original/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-opus-5-5--r01/plugin/original/bottom.png" }, "originalImageHashes": { "perspective": "ab1538f1c48cd2770633018012b3fa008e754646506da5168c704e24b3dd96ea", "front": "0bf5030cd0b6af7e0dd51140fb926facb9d4d25420893e2d811cacbbcf790ced", "back": "cd000477a4fd8c9c934dd1073c99505f06ad53ef5d9d6d51b3fc32d17e53fb31", "left": "fbe3fe35aefe852cf74f7723c3e14b04655fda93926b5ea6f96ea23d5caa8dc8", "right": "57d6efb253409da210e3e3a23fba81310cd887b48f91b6944ebe9463ab3e9f5b", "top": "5eb2f57988de8b13320f8b8e1840bdb9a2137b273adbe8897278469e58f12054", "bottom": "154d126e0aee2f7948ff87322472d648c3b2ed841c6ddbf1992055db5428ae4e" } }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "note": "One review noted: The plugin result provides clearer articulated construction, visible hinge spacing and stronger material separation across views. The no-plugin result has a polished finish, but its minimal joint hardware leaves two critical mechanical relationships unverified.", "criteria": [ { "id": "joint_continuity", "label": "joint continuity", "question": "Are base-to-arm, elbow, and arm-to-shade connections physically continuous through credible hinge hardware?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "hinge_clearance", "label": "hinge clearance", "question": "Do adjacent arms and hinge cheeks retain readable mechanical clearance without unintended intersections?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "shade_cavity", "label": "shade cavity", "question": "Is the shade mouth open with a distinct recessed bulb that does not pierce the shell?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cable_contact", "label": "cable contact", "question": "Does the cable meet its socket and rest on the desk plane without floating or being buried?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "base_contact", "label": "base contact", "question": "Does the weighted base sit flat on the supporting plane?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "finish", "label": "finish", "question": "Do enamel, brushed metal, rubber and reverse-side construction details read as a finished product?", "critical": false, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "rawReview": { "taskId": "task_lamp_clearance_holdout", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "current", "current" ], "criterionMajorities": [ { "criterionId": "joint_continuity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "hinge_clearance", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "shade_cavity", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cable_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "base_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "joint_continuity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "hinge_clearance", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "shade_cavity", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cable_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "base_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "current" }, "decodedWinner": "current", "scoresByMode": { "vanilla": { "taskFidelity": 7, "silhouetteAndProportion": 7, "constructionPlausibility": 6, "craftsmanshipAndDetail": 6, "materialsAndReadability": 7, "surfaceFinishAndShading": 8, "materialTextureQuality": 6, "lightingAndPresentation": 7, "finalStageCompleteness": 7, "multiviewConsistency": 8 }, "current": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "result": { "winner": "B", "confidence": 0.88, "scores": { "A": { "taskFidelity": 7, "silhouetteAndProportion": 7, "constructionPlausibility": 6, "craftsmanshipAndDetail": 6, "materialsAndReadability": 7, "surfaceFinishAndShading": 8, "materialTextureQuality": 6, "lightingAndPresentation": 7, "finalStageCompleteness": 7, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "joint_continuity", "A": "unclear", "B": "pass", "evidenceA": "Connections appear continuous, but the tiny elbow pin and minimal arm-end fittings do not clearly establish credible hinge construction.", "evidenceB": "Distinct brackets, pivot hardware and collars visibly connect the base, both arms and shade." }, { "criterionId": "hinge_clearance", "A": "unclear", "B": "pass", "evidenceA": "Arm ends meet at very small joints; separation between moving components cannot be resolved.", "evidenceB": "Front and top views show spaced hinge cheeks and narrower connecting members with readable gaps." }, { "criterionId": "shade_cavity", "A": "pass", "B": "pass", "evidenceA": "Underside view shows an open reflector cavity surrounding a distinct inset bulb without visible shell penetration.", "evidenceB": "Underside view reveals the open shade interior and a distinct bulb contained within the mouth." }, { "criterionId": "cable_contact", "A": "pass", "B": "pass", "evidenceA": "Cable visibly meets the base edge and follows the supporting plane in perspective and side views.", "evidenceB": "Cable emerges through a visible base-side fitting and lies along the desk plane toward the plug." }, { "criterionId": "base_contact", "A": "pass", "B": "pass", "evidenceA": "Low round base has a flat underside and visibly sits against the supporting plane.", "evidenceB": "Broad round base and dark lower pad sit flush against the supporting plane." }, { "criterionId": "finish", "A": "pass", "B": "pass", "evidenceA": "Smooth red enamel, contrasting metal fittings and dark cable and underside read as finished, though mechanically sparse.", "evidenceB": "Smooth turquoise enamel, differentiated metal hardware, dark cable, underside pad and rim detailing give a coherent product finish." } ], "majorDefects": { "A": [ "Elbow and arm-end hardware are too simplified and small to verify mechanical clearance.", "Thin uniform arms and sparse fittings weaken construction readability." ], "B": [] }, "rationale": "B provides clearer articulated construction, visible hinge spacing and stronger material separation across views. A has a polished finish, but its minimal joint hardware leaves two critical mechanical relationships unverified." } }, { "mapping": { "A": "current", "B": "vanilla" }, "decodedWinner": "current", "scoresByMode": { "current": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.3, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.8, "lightingAndPresentation": 8, "finalStageCompleteness": 8.4, "multiviewConsistency": 8.8 }, "vanilla": { "taskFidelity": 7.3, "silhouetteAndProportion": 7.5, "constructionPlausibility": 6.4, "craftsmanshipAndDetail": 6.6, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 7.4, "lightingAndPresentation": 7.8, "finalStageCompleteness": 7.2, "multiviewConsistency": 8.6 } }, "result": { "winner": "A", "confidence": 0.86, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.3, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.8, "lightingAndPresentation": 8, "finalStageCompleteness": 8.4, "multiviewConsistency": 8.8 }, "B": { "taskFidelity": 7.3, "silhouetteAndProportion": 7.5, "constructionPlausibility": 6.4, "craftsmanshipAndDetail": 6.6, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 7.4, "lightingAndPresentation": 7.8, "finalStageCompleteness": 7.2, "multiviewConsistency": 8.6 } }, "criterionResults": [ { "criterionId": "joint_continuity", "A": "pass", "B": "unclear", "evidenceA": "Visible brackets and pivot hardware connect the base, both arm segments, and shade.", "evidenceB": "Connections appear continuous, but the tiny elbow node does not clearly establish a credible hinge assembly." }, { "criterionId": "hinge_clearance", "A": "pass", "B": "unclear", "evidenceA": "End-on views expose separated bracket cheeks and narrower connecting members with readable gaps.", "evidenceB": "Elbow and shade hardware are too reduced to establish clearance between moving members and hinge cheeks." }, { "criterionId": "shade_cavity", "A": "pass", "B": "pass", "evidenceA": "Underside shows an open pale interior and a distinct bulb contained within the mouth, without visible shell penetration.", "evidenceB": "Underside shows a separate bulb recessed within the open bell interior, without visible shell penetration." }, { "criterionId": "cable_contact", "A": "pass", "B": "pass", "evidenceA": "Cable visibly meets the base-side fitting and follows the supporting plane in side views.", "evidenceB": "Cable meets the base edge and lies along the desk plane in opposing side views." }, { "criterionId": "base_contact", "A": "pass", "B": "pass", "evidenceA": "Rounded weighted base has a flat lower contact surface seated against the desk.", "evidenceB": "Round base sits flush against the supporting plane with a consistent contact shadow." }, { "criterionId": "finish", "A": "pass", "B": "pass", "evidenceA": "Smooth turquoise enamel, contrasting metal hardware, dark cable, underside pad, and visible fasteners read as a finished product.", "evidenceB": "Smooth red enamel, metallic fittings, dark cable, pale shade interior, and underside pad establish a finished, though sparsely detailed, product." } ], "majorDefects": { "A": [], "B": [ "Elbow articulation is visually underdeveloped; hinge construction and mechanical clearance remain unproven.", "Uniform thin arms and minimal hardware weaken the construction hierarchy." ] }, "rationale": "A wins through visibly developed hinge brackets, readable clearance, and stronger mechanical detailing. Both show polished finishes, open shades with contained bulbs, and seated bases and cables; B provides weaker evidence for functional articulation." } } ] }, "id": "task_lamp_clearance_holdout--claude-opus-5-5--r01", "repetition": 1, "cohort": "October 2026 spatial pilot", "limitation": "One generation per condition; two counterbalanced Astra judgments of the same images. Vanilla runs were collected later and paired with the existing frozen plugin runs; generation order was not counterbalanced. With plugin means pinned skill workflows and bundled scripts, not an MCP ablation. Concurrent times are not a latency benchmark. Structural scores are proxies, not visual quality or proof of contact.", "judgeCount": 2 }, { "id": "tabletop_press--claude-opus-5-5--r01", "task": "tabletop_press", "taskTitle": "Tabletop lever press", "model": "claude-opus-5-5", "modelTitle": "Opus 5.5", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "tabletop_press", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "vanilla", "vanilla" ], "criterionMajorities": [ { "criterionId": "press_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_drive_chain", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_supports", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "press_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_drive_chain", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_supports", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.4, "craftsmanshipAndDetail": 8.9, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.9, "multiviewConsistency": 9 }, "plugin": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 8.3, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 8, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.1, "multiviewConsistency": 9 } }, "result": { "winner": "A", "confidence": 0.79, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.4, "craftsmanshipAndDetail": 8.9, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.9, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 8.3, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 8, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.1, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "All required components are visible, including the guided central drive, ram, brass die, top spring, and side tray.", "evidenceB": "All required components are visible; exposed rack teeth particularly help identify the drive." }, { "criterionId": "press_drive_chain", "A": "pass", "B": "pass", "evidenceA": "Lever pivot crosses the drive housing at the guided vertical mechanism; the ram sits over the die.", "evidenceB": "Lever enters a pivot housing beside the toothed ram, which aligns over the die; the internal gear is concealed plausibly." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "Spring is seated beneath its cap; tray rests on the base, and bolted feet and a substantial gusset support the frame.", "evidenceB": "Spring has visible seats, tray rests on the base, and the upright and ram have visible mounting and guide structures." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Beveled surfaces, differentiated metals, fasteners, tooling mounts, and rubber feet create a convincingly finished assembly.", "evidenceB": "Smooth paint, bright metal, contrasting tray, bevels, and fasteners establish a finished prop, though detailing is simpler." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "A wins through stronger structural reinforcement, richer tooling and mounting detail, and clearer material hierarchy. B presents a readable rack mechanism and consistent smooth finish, but its simpler base, slender frame, and broad plain components feel less fully refined." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 8, "silhouetteAndProportion": 7, "constructionPlausibility": 7, "craftsmanshipAndDetail": 6.5, "materialsAndReadability": 7, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 9, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8, "finalStageCompleteness": 8.5, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.91, "scores": { "A": { "taskFidelity": 8, "silhouetteAndProportion": 7, "constructionPlausibility": 7, "craftsmanshipAndDetail": 6.5, "materialsAndReadability": 7, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 9 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 9, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8, "finalStageCompleteness": 8.5, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "Base, upright, lever and pivot, exposed rack teeth, rectangular ram, brass die, top spring, and shallow tray are visible.", "evidenceB": "All required parts are readable, including the exposed rack, pivot axle, stepped ram, brass die, top spring, and collection tray." }, { "criterionId": "press_drive_chain", "A": "pass", "B": "pass", "evidenceA": "Lever enters a pivot housing beside the toothed ram, which is aligned over the die; internal engagement is concealed.", "evidenceB": "Lever pivot and transverse axle meet the exposed rack assembly, with the guided ram centered above the die." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "Spring seats on the upper assembly; tray and die rest on the base; frame has visible mounting feet and fasteners.", "evidenceB": "Spring is seated and capped; tray and die have base support; bolted feet and a substantial gusset reinforce the upright." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Painted surfaces, metallic ram, brass components, rounded grip, and softened edges establish a finished prop, though detailing is comparatively basic.", "evidenceB": "Beveled housing, reinforced frame, detailed fasteners, stepped metal tooling, contrasting base, and rubber feet produce a convincing manufactured finish." } ], "majorDefects": { "A": [ "Slender upright and minimal bracing make the load-bearing frame less convincing.", "Broad plain panels and simplified tooling reduce industrial refinement." ], "B": [] }, "rationale": "B provides a clearer drive assembly, stronger structural reinforcement, and more deliberate tooling and mounting details. Its contrasting materials and refined edges sustain a finished industrial appearance across all views. A meets the requirements but looks simpler and less structurally substantial." } } ] }, "votes": { "baseline": 2, "candidate": 0, "tie": 0 }, "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 188, "triangles": 19432, "images": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "54b2a0e0e906eae37381c2a2f78bd7a93dd340746353ab2b2552f242eeb5b56d", "front": "ab7ea1a72515ff56565d3bf55d72f7ad4999dad7ddb36d78b132d922051db2e2", "back": "fecba6a1204b618b97ccea5e12bdaa9e7afc138e8e5322d60d2a3d5a8e44ec5a", "left": "a984d6684a284f2df136ba4fdf91c14912c12cac933cbbac64725b5b9f6a87f7", "right": "cec96b5026bb4a09c9e922954c69b9284f2277adab701092e2d5d1163658dfbb", "top": "e934a3ae64365e4b4f9683d346e870501523d67a57a72d27c8edc2340a2e2384" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "3992681e389fe99d0e2e70417a7800e2e9bb54c168894fdae429da6c33beeeb3", "front": "8676ef059b34faa50ad9f8a3b15feffe52e5ec0c0be01a7c3d47710466ee8a3b", "back": "7d3c84c834392fc04b882dbd97898c12b6dac9713ec9becec479e52a7e47ddd4", "left": "c7e848ee6bcf123f3a3cdc95d49b0338d22f5dd7311c3d3d927ebfc820aed69c", "right": "e48f3d659ca6352837d9adba9a9f0900a94ec7c4445df43448c09e535f2cb3d0", "top": "142768ed13d42a104f6b5432eea885107d710622287c452d0d6bceb1cd013732" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "676edc8672bedeac719fd47bcac38c29c412cfd8ad175871154fd2f8ab69d638", "front": "5d5711ce31d4947ae8ab2724eb484410abb62105f87bab805b92ac4b6b1802fb", "back": "17bae3234d33b132ce8214b0ab5feda9e1b74fc4f884e1ed7d998c2c07b31ca8", "left": "7cc79e72eb68e0bdc4aa9e382ca4b6dca246720dae5e434381720747fc67c776", "right": "be5f91d918f2c73ee923b5fc15d2a51c99879d0408021cd409e557cfa749dd19", "top": "c36b21270e40c2d3da98a6d6cd42621f312afb6bbdc7608d103f2f7aa50cf778" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "1ab680e9654307e7179b288d56f68c7ef93d5c43676538a86a63976b0bc8afec", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "1ab680e9654307e7179b288d56f68c7ef93d5c43676538a86a63976b0bc8afec" }, "artifactHashes": { "create_asset.py": "b655beca28a0805fe9ec1722821ea4539a4af88713b13203c66a0c524e3ea9f4", "asset.blend": "1ab680e9654307e7179b288d56f68c7ef93d5c43676538a86a63976b0bc8afec", "asset.glb": "3a0ca4241bd43db9a223f1534c7fb6f0c6ac307c56f87f4ac0d1c32e5ff98223" }, "failedChecks": [], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "e147da73ad0c4c84e79c127ae9eb1e37b1a299e26700ddfb96d17df5c0172260", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 5 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 195, "triangles": 19508, "images": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "9cb8896a768ad98c5c6188c808431714e794a8d4616c124f7b18f726477e2b02", "front": "331459634f773971198347c71d0f059150662a4bffe17fbe191927ed707cb35b", "back": "d120825a80eed23e6f8a1e3637cca9c8f9b75ed210624ac16415d915715a8f0a", "left": "984a0129f77242921508b638e306d754252cb8b726a13655b293ed2ec1f873f7", "right": "490012fe1f8a7c0f4f1215ef9dff6cf40fd4f1a40413588383ce9597aa85bd2f", "top": "4a35becdb746f4f095bf943a14de8009ce99681a4e20ce9fd9818ed1e2620f0d" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "5380fcda133ea310294721ea24ee6e4e51fc4646446338204a47c14f82b809be", "front": "4ff6ba10043194bae43daea12e5230df14cde4fea7d61fb36fa4208fd6ef18f3", "back": "e72a983cae7e74362bd3c2082dca134bbd5ab723a03a24bbde71292caafd3918", "left": "7097c2ba7ba4b9cae145e7782168b2667beb7825203335a4959394db5c258795", "right": "7d98f642f357a07d3a75356964c89ed2bd0eb00490a26f87cedecd8f2d1c097d", "top": "53b4bea0ed48cb82d2e453581148c8bf9e1733b1c5b41331cbab8dc85e073803" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-opus-5-5--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "d2333a62cf625e2b72ae64a27104fe3ab0f286bbb4771e291b6c5db6a17578d5", "front": "18c21f1ca3cdd5f4fc7934413e96df92b26e82a40d8054c86f12a77e428e8227", "back": "5f7e46bb95d0ce7cacfb933e24efee60117b58144b7d2dcc07471981ab053ee4", "left": "d34762d8c68b8574f92297886e90c8cfca1b37b96dc8a1e902d42fa3a0c0231f", "right": "28356fc082b182ee511cd46533286b819db6e3a85e59b20bdd382f0c5101c22a", "top": "4da679c1406690a3698ded1a5cf7c2b857995033723a992b6389cabbd869b1db" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "03d464fe8d975bb7d8acdffee2316442342d77fed5dfe7e2d8fab621bd99bfe6", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "03d464fe8d975bb7d8acdffee2316442342d77fed5dfe7e2d8fab621bd99bfe6" }, "artifactHashes": { "create_asset.py": "c1f77f6b7ceba96e7f96ee80a0df42dcd2710b1fce072ce31be5c22f24c8096e", "asset.blend": "03d464fe8d975bb7d8acdffee2316442342d77fed5dfe7e2d8fab621bd99bfe6", "asset.glb": "50f35f210e24973782bfc2c77b073ddda4e1776ca30717c901c41bc962c38869" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "e147da73ad0c4c84e79c127ae9eb1e37b1a299e26700ddfb96d17df5c0172260", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 5 }, "criteria": [ { "id": "press_parts", "label": "press parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_drive_chain", "label": "press drive chain", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_supports", "label": "press supports", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_finish", "label": "press finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The no-plugin result wins through stronger structural reinforcement, richer tooling and mounting detail, and clearer material hierarchy. The plugin result presents a readable rack mechanism and consistent smooth finish, but its simpler base, slender frame, and broad plain components feel less fully refined." }, { "id": "winch_drawbridge--claude-opus-5-5--r01", "task": "winch_drawbridge", "taskTitle": "Animated miniature winch drawbridge", "model": "claude-opus-5-5", "modelTitle": "Opus 5.5", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. This gallery shows still views; reviewers also received sampled animation frames when available. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "winch_drawbridge", "repetition": 1, "baselineAutomatedScore": 92, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "plugin", "vanilla" ], "criterionMajorities": [ { "criterionId": "bridge_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_hinge", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_cables", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "bridge_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_hinge", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_cables", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.3, "lightingAndPresentation": 8.4, "finalStageCompleteness": 7.8, "multiviewConsistency": 8.6 }, "plugin": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.3, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 7.5, "lightingAndPresentation": 8.4, "finalStageCompleteness": 8.4, "multiviewConsistency": 8.6 } }, "result": { "winner": "B", "confidence": 0.77, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.3, "lightingAndPresentation": 8.4, "finalStageCompleteness": 7.8, "multiviewConsistency": 8.6 }, "B": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.3, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 7.5, "lightingAndPresentation": 8.4, "finalStageCompleteness": 8.4, "multiviewConsistency": 8.6 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two capped towers, planked deck, upper crossbeam and drum assembly, side crank, hinge hardware, and two lifting lines are visible.", "evidenceB": "Two capped towers, crossbeam, hinged deck, separate low-mounted drum with axle supports and crank, and paired lifting cables are visible." }, { "criterionId": "bridge_hinge", "A": "pass", "B": "pass", "evidenceA": "The three poses retain the deck root at the visible tower-base hinges while the free end rises to vertical.", "evidenceB": "The deck pivots from horizontal through an inclined pose to near vertical around the same visible base hinge." }, { "criterionId": "bridge_cables", "A": "pass", "B": "pass", "evidenceA": "Paired lifting lines connect the upper winding assembly to the deck corners and shorten consistently as the deck rises; the raised deck partly occludes them.", "evidenceB": "Paired cables route from the low drum through upper guides toward the moving deck. Visible segments remain aligned with their attachments across the supplied poses." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Consistent pastel materials, softened masonry edges, deck framing, and metal hardware produce a finished miniature, though detailing is relatively simple.", "evidenceB": "Coherent softened edges, metal collars, drum supports, deck trim, and restrained tower details remain consistent across views and poses." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins narrowly through its more legible separate winch, supported drum axle, cable routing, and finer construction details. A presents convincing hinge poses but has a less differentiated upper mechanism and chunkier detailing. Both satisfy the visible critical states. Continuous timing, loop smoothness, flicker, and between-frame collisions remain unclear." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 8.4, "silhouetteAndProportion": 8, "constructionPlausibility": 7.7, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8, "materialTextureQuality": 6.9, "lightingAndPresentation": 8.2, "finalStageCompleteness": 7.7, "multiviewConsistency": 8.4 }, "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.8, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.4, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.6, "multiviewConsistency": 8.8 } }, "result": { "winner": "B", "confidence": 0.87, "scores": { "A": { "taskFidelity": 8.4, "silhouetteAndProportion": 8, "constructionPlausibility": 7.7, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8, "materialTextureQuality": 6.9, "lightingAndPresentation": 8.2, "finalStageCompleteness": 7.7, "multiviewConsistency": 8.4 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.8, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.4, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.6, "multiviewConsistency": 8.8 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two towers, crossbeam, planked deck, ground-level drum, crank, hinge hardware and two lifting lines are visible.", "evidenceB": "Two towers, crossbeam, elevated drum, crank, deck hinges and two chain-like lifting lines are clearly visible." }, { "criterionId": "bridge_hinge", "A": "pass", "B": "pass", "evidenceA": "The three poses retain the deck's near edge at the visible base hinge while its free end rises.", "evidenceB": "Visible hinge brackets anchor the same deck edge in lowered, intermediate and upright poses." }, { "criterionId": "bridge_cables", "A": "pass", "B": "pass", "evidenceA": "Paired lines visibly route from the low drum through upper guides toward the deck; the sampled poses show plausible retained connections.", "evidenceB": "Both lifting lines connect the upper winch assembly to the deck's outer corners and shorten plausibly as the deck rises." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Consistent pastel materials, smooth drum hardware and restrained edge detailing produce a finished miniature, though tower and deck detailing remain simple.", "evidenceB": "Consistent tower trim, articulated lifting lines, hinge plates, drum collars and underside deck reinforcement give the assembly a more resolved finish." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins through clearer hinge construction, better integrated winch hardware and more substantial deck reinforcement. A satisfies the visible requirements but has simpler detailing and less legible cable routing near the raised deck. Continuous timing, loop smoothness, flicker and between-frame collisions remain unclear for both." } } ] }, "votes": { "baseline": 1, "candidate": 1, "tie": 0 }, "vanilla": { "score": 92, "rawScore": 92, "hardGate": true, "rawHardGate": true, "seconds": 300, "triangles": 3864, "images": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "cf2f4c82cd2b68ce151163eb1e1afa4bacf05ac0040144bb81ef80a0c89dee2a", "front": "39e7318de26f20a9dc5ec75fe78ed2541af046e6bd95b85d565917562ac40c82", "back": "65f44b37f7c4633ec40a8e9fa6a610182bec43b74e010670ae3e56690fe59258", "left": "161619a6c8fd935f84fe3ea7c11921b2b7eb153f30e31414d31de884622542f3", "right": "29b21c791c5ec7d747fbc33876c204463dec11a617dbb567572a730053b2cba0", "top": "0d7a0b7689d317b75ada4f4bfa70acab3feffbbd680ab36b01683a3485d5378a" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "996d5402a3eb10e210b172923630a2a371b606a6f5a24a1c6d8c57e5b205cf62", "front": "e919c6320008fc2b223b98ff3df813e7d6cf36d189ef58b37fefe74f9b3ddc05", "back": "62b027661b0ec4b30834d4f4973222f481d7b2986c05383ff3742040d1240aee", "left": "11ae640fe5587ac5322f06058e22f29a46accd6dc1463ed55cda872fb8904323", "right": "64be019b08dd987ee31857a53f3c07b3e879a602959da6e04076f39ace7434a1", "top": "beb6545b2c52e570a0fde241bda6c48c18248f22df8b8b0601c2db89575a116d" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "11c78aeecb47161f926424ba173a58b88667729d98fbd0246e78c2b9959fb8d4", "front": "0cd89683e9389e4f1511e5f5dbf456e860bd4408506aa18f0938bba2d7f5fecf", "back": "cd3cd179ce960e82417ed081a52c3c1f533cb0f2e52f4d42022e63585f97fbb3", "left": "d93c32f365e9a7a844cd561730f9e99182da0a58c44f05d4b3c5b8aa47576618", "right": "20ba9f991dd66358e7ebc5238056fd79a2234cf58d554bc94358ad4a864475b3", "top": "e6b16c5e740b593cb406994a805c0c773e883bc53c7f427be82a9241538e191a" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "cc4b73fad7109e55e8b9c3c2bc2d556e60f47d6e78d20d92eb418d5017f9cf15", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "cc4b73fad7109e55e8b9c3c2bc2d556e60f47d6e78d20d92eb418d5017f9cf15" }, "artifactHashes": { "create_asset.py": "385822b2b5976acf98d67a39339c420e2625d82b0f50aa650eb51d611da0f56e", "asset.blend": "cc4b73fad7109e55e8b9c3c2bc2d556e60f47d6e78d20d92eb418d5017f9cf15", "asset.glb": "e551768cf18d5ae7ad0f95918f9597736d283bb14281b2dd90bfa4fcbfe8c43d" }, "failedChecks": [ { "id": "uv_coverage", "passed": false, "earned": 0, "possible": 4, "detail": "0/79 meshes have UV layers; minimum ratio 0.55" }, { "id": "finish_shading_profile", "passed": false, "earned": 0, "possible": 4, "detail": "0.0% smooth polygons for polished_smooth" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "1a47e1548eb2cabc9674cee1f96b10604b5692d3aa3d237101240d3298d40661", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 5 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 219, "triangles": 8336, "images": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "17db2cc23951a752371f130154e01f286314161580c427dbcdacd8669b5c2451", "front": "cb317dc25ce77f5fb212f1c97e71a0270e1770f2af12b7517b9d1ed57c8871d3", "back": "06f4bdcd0a15091a7b8e8a5c0719f5bcfb88d693875a74a0817b83c16641fb7d", "left": "32d001e57ceee9b1a08fb38f8583b11546fd6903e6a06a0761fcfeaac86f0172", "right": "107e99e2c3a4d8ec1b9d89dd8cb4a28531dd5fc716a62d4be6f847e919c17dad", "top": "cd7d98f5f6c828c5ea3d12287b4948938a2ad873f71518da77ef89e47addf736" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "eda4f5aa00c8d82ba18c19d63c65a347255d0dbdb9b66c2399d432389436ebae", "front": "741905b22f010ceab301724e06cc35fae4fd9842eb706e1a546f423833d3746f", "back": "14ef074e7db0bea180ab43c97bef0a8de129f3439d5273a0863402f9bcac17e7", "left": "389ca1875c3a909a2d376e44eb59657a01e59a8a3e5df6fc6a1c52dcad9a309d", "right": "9f2f8cd53aa57fae7fe1950efa4b9e74f46a0593957c63a74e694785aece2d50", "top": "2114436718d5b469d6b78efb0f648e5bbc26a7af12208e17aa745b7f7521c687" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-opus-5-5--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "09429007ce12963cd41bbcc21889692e5cdd060c76280ba96c292a29e8aa9ee2", "front": "3ae4473f9bc04ebc8430c6d4982b5ab73973430dca9b5431f17c46c7d2d79441", "back": "f61b13f7e59684cf4ee8e135f837b953aed37df96cb959f7840e6f57cef24ee2", "left": "77b72ef7c7125d8bfb5c7ab2dab50a054ca203f1b7ef568eb22e1660a828593a", "right": "9b413b768d94d27a815a27c7afd7650281166f55ab0ac2f74418878cafe7b648", "top": "aa7c0e3245c1e4dc7733c29c5e9a13d5b6654a5605414f923d67ce1745b1618a" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "4243906f1314552c1cbd7bd29592bf2a138599a064746afc7920e420486d7eea", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "4243906f1314552c1cbd7bd29592bf2a138599a064746afc7920e420486d7eea" }, "artifactHashes": { "create_asset.py": "81899357abf4923be20650fe0fd66d2ec56e8a9c609ef3d93a3acd2944502da7", "asset.blend": "4243906f1314552c1cbd7bd29592bf2a138599a064746afc7920e420486d7eea", "asset.glb": "d8f1c139abb50e036b31dfa1f541759670ec7fc5466267705640b3b47e7153bd" }, "failedChecks": [], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "1a47e1548eb2cabc9674cee1f96b10604b5692d3aa3d237101240d3298d40661", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 5 }, "criteria": [ { "id": "bridge_parts", "label": "bridge parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "bridge_hinge", "label": "bridge hinge", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "bridge_cables", "label": "bridge cables", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "bridge_finish", "label": "bridge finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The plugin result wins narrowly through its more legible separate winch, supported drum axle, cable routing, and finer construction details. The no-plugin result presents convincing hinge poses but has a less differentiated upper mechanism and chunkier detailing. Both satisfy the visible critical states. Continuous timing, loop smoothness, flicker, and between-frame collisions remain unclear." }, { "task": "joinery_stool", "taskTitle": "Splayed-leg joinery stool", "model": "claude-sonnet-5-5", "modelTitle": "Sonnet 5.5", "vanilla": { "score": 90.94, "rawScore": 90.94, "hardGate": false, "rawHardGate": false, "seconds": 65, "triangles": 5548, "images": { "perspective": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/hd/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/hd/bottom.png" }, "imageHashes": { "perspective": "76f5ffde4a92eb3a1c3e2f1f5f387e347649bf98f93a31b86d68f37c7948070d", "front": "ef4def763f127597781cafbf3c4be3fc7b88aca9766a9b5b15e46c5c8879fd2f", "back": "2b27d7e561d1a7f4f0e25d5e65f4af16efae3872738e4db4acdaac1cd75e36a6", "left": "2f88a0f6b33e6c02ca90ed68a8ac76aa6944e6ed7e306bd669bf732a92fcb202", "right": "22a2d4ef9626015537fe4261e9e020e4d748b6a8c98574d8612341bc7b30be9f", "top": "0c391f8fa5d5a3f2808442cba85cc110853478f4f259a21224e291dc2da0687c", "bottom": "717eb42f5f5d0faa9193e3c9d4e7147f0f6d007ac0787992c4a3351737c2aed2" }, "failedChecks": [ { "id": "semantic_part_coverage", "passed": false, "earned": 18.75, "possible": 25, "detail": "3/4 required semantic part groups found" }, { "id": "material_count", "passed": false, "earned": 3.333333333333333, "possible": 5, "detail": "2 materials; minimum 3" }, { "id": "uv_coverage", "passed": false, "earned": 2.8571428571428568, "possible": 4, "detail": "12/21 meshes have UV layers; minimum ratio 0.80" } ], "artifactHashes": { "create_asset.py": "9a86ed9c446758194a9ea1df1fc820bdbffa173166354835a92d0192603cfbe4", "asset.blend": "39486e4d1f782045016a2c87d1316dc385cd54a3e42f82fd6cd42017eae38054", "asset.glb": "5495afd2b1475c9d5acea4c9c3912b57a04c10829c95b5439f5ec91fd64d0d7d" }, "generationTaskFingerprint": "28c3c6ad61d6de6e939213c55613d617296ab05d5b926a06521a998c7ea82fe8", "correctedTaskFingerprint": "2cbfc0d0f305e997e246fcd1d18761664f42e38cc53f721e487550d43101381f", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "agentVersion": "2.1.287 (Claude Code)", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": null, "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "baseline", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "39486e4d1f782045016a2c87d1316dc385cd54a3e42f82fd6cd42017eae38054", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/original/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/vanilla/original/bottom.png" }, "originalImageHashes": { "perspective": "108464156c4ac0bc114579d86c8e9af0fefc2e8713d3788b05ddf9e3a6d8c203", "front": "cf6df28321c573082bb0985fef9b27c862d23c47bdd08175efb6105cfab854d8", "back": "00be22baeec8850b8f80613675448448d40734871fcf5f745737cff2177811c2", "left": "358ff06d6bbae31c62ce64a9f928666ad86e717f0f25d78a6047d149136382df", "right": "3be8aa994ce772e90c6e424e11ccd90c37b18cf4d881c25dc758c9c3c4bbb5f4", "top": "075df7f22132ed39712f72c484857cc13826e24a3ae6bc0964d80647fd843719", "bottom": "24e347ea8baa06b99f75de4f5de0afadfc3e570f63249345caeaf1b1a8ec7380" } }, "plugin": { "score": 96, "rawScore": 89.75, "hardGate": true, "rawHardGate": false, "seconds": 102, "triangles": 4844, "images": { "perspective": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/hd/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/hd/bottom.png" }, "imageHashes": { "perspective": "3705d1107f5bd3c0547d1a64e1a10a8b58d0dc0737fcefd2667cf26f086b0872", "front": "8f6027bff6afd7f17fe68153787392443a26a3e7993d74500ea11a24f0152137", "back": "83987fd8da4fd58831bf9dace47097f1e7494ca8a9275113665121838b21ca81", "left": "6232225e88528cc90a9913051c5918da698b285d0a10ab29ad3d4a47c1fe1e4a", "right": "4ece1da15ac7ccd1978c248f5ec628ccf4786f9297505891700bc4955d746169", "top": "b36184fea6c260571a8f7da23b4585d15113bee475e58a0c4ce1efd58df38bee", "bottom": "ad5ce7794a4488083a89fb827dae74e4faf646a8d8d0f5443b8f1732803966a6" }, "failedChecks": [ { "id": "uv_coverage", "passed": false, "earned": 0, "possible": 4, "detail": "0/21 meshes have UV layers; minimum ratio 0.80" } ], "artifactHashes": { "create_asset.py": "1dcab36c117b696b9d8c0cb8747e5d6a04b92063907adc1c63537d8a60b98b2b", "asset.blend": "4c5effca479ab022832e1089f9f24b3737257a16ff2823ef9d8df0d975b4af70", "asset.glb": "572cdd8160db1cf48fae50b8446f8bda53b832ee781edda6ec516c2a3f666a8e" }, "generationTaskFingerprint": "28c3c6ad61d6de6e939213c55613d617296ab05d5b926a06521a998c7ea82fe8", "correctedTaskFingerprint": "2cbfc0d0f305e997e246fcd1d18761664f42e38cc53f721e487550d43101381f", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "agentVersion": "2.1.287 (Claude Code)", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "skills", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "4c5effca479ab022832e1089f9f24b3737257a16ff2823ef9d8df0d975b4af70", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/original/top.png", "bottom": "benchmarks/gallery-matrix/joinery_stool--claude-sonnet-5-5--r01/plugin/original/bottom.png" }, "originalImageHashes": { "perspective": "fc8573087a4c33077f49a669794521a8eb7c2a4dc1f66a37c322fb4872620970", "front": "b5dd9ce15b84c55f94a431b2aa23bfe5f456c9386df1f0253412cc55ea147a5a", "back": "95ae4f55f348a667111a6fc81248cd88b25e09ed53fc5fac7d74d4c3845cdb7c", "left": "1ef22012ae319c79961907e4fb1268f17046f9d9e80b81ee879648355d043084", "right": "14afdf9870ece62dd03f8e41196ce561ead05f3ea1a11b8ecfd73930b1c36e13", "top": "a4b66912f0eb15e0a12b9056ae5d84493250fce6caf7a48383395e7c8b9c273e", "bottom": "4473de6dc2225cd982216e5d4de1dfea420578dfb734a534bdc98b5545ba6ebb" } }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "note": "One review noted: The plugin result wins through its clean rounded rectangular seat, controlled edge refinement and more finished silhouette. Both show connected assemblies and preserved openings, but the no-plugin result's notched seat and repetitive grain substantially reduce fidelity and polish. Absolute height cannot be verified from these images.", "criteria": [ { "id": "seat_support", "label": "seat support", "question": "Do all four leg tops meet the seat underside without open seams or piercing its upper surface?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "stretcher_joints", "label": "stretcher joints", "question": "Do all eight stretcher ends meet their legs without floating gaps or exposed protruding ends?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "ground_support", "label": "ground support", "question": "Do all four feet terminate on the same plane without visibly hovering or being buried?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "negative_space", "label": "negative space", "question": "Are the spaces under the seat and inside the stretcher ring intentionally open and free of accidental crossing geometry?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "assembly_count", "label": "assembly count", "question": "Are exactly four legs and four stretchers visible across the views?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "finish", "label": "finish", "question": "Are curved edges, wood finish and inset hardware coherent on reverse sides and underside as well as the hero view?", "critical": false, "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 1, "fail": 1, "unclear": 0 } } ], "rawReview": { "taskId": "joinery_stool", "repetition": 1, "baselineAutomatedScore": 90.94, "candidateAutomatedScore": 89.75, "hardGates": { "baseline": false, "candidate": false }, "visualWinnerVotes": [ "current", "current" ], "criterionMajorities": [ { "criterionId": "seat_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "stretcher_joints", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "ground_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "negative_space", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "assembly_count", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "finish", "baselinePasses": 0, "candidatePasses": 1, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "seat_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "stretcher_joints", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "ground_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "negative_space", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "assembly_count", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "current" }, "decodedWinner": "current", "scoresByMode": { "vanilla": { "taskFidelity": 6, "silhouetteAndProportion": 6, "constructionPlausibility": 7, "craftsmanshipAndDetail": 5, "materialsAndReadability": 6, "surfaceFinishAndShading": 5, "materialTextureQuality": 3, "lightingAndPresentation": 8, "finalStageCompleteness": 5, "multiviewConsistency": 8 }, "current": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7, "materialsAndReadability": 6, "surfaceFinishAndShading": 8, "materialTextureQuality": 5, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.94, "scores": { "A": { "taskFidelity": 6, "silhouetteAndProportion": 6, "constructionPlausibility": 7, "craftsmanshipAndDetail": 5, "materialsAndReadability": 6, "surfaceFinishAndShading": 5, "materialTextureQuality": 3, "lightingAndPresentation": 8, "finalStageCompleteness": 5, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7, "materialsAndReadability": 6, "surfaceFinishAndShading": 8, "materialTextureQuality": 5, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "seat_support", "A": "pass", "B": "pass", "evidenceA": "Underside and elevation views show leg tops contacting the seat, with no visible open seams or upper-surface penetration.", "evidenceB": "All four leg tops visibly terminate against the seat underside; no open seams or piercing are apparent." }, { "criterionId": "stretcher_joints", "A": "pass", "B": "pass", "evidenceA": "The hero and underside views show four rails ending against the legs without visible floating ends or exposed extensions.", "evidenceB": "Four stretchers visibly connect to the legs; the complementary views show no detached or protruding rail ends." }, { "criterionId": "ground_support", "A": "pass", "B": "pass", "evidenceA": "Feet align with the floor in the elevations, without visible hovering or burial.", "evidenceB": "Front and side elevations show level foot terminations and consistent floor contact." }, { "criterionId": "negative_space", "A": "pass", "B": "pass", "evidenceA": "The space below the seat and within the stretcher ring remains open, with no accidental crossing members.", "evidenceB": "Under-seat openings and the ring interior are clear and unobstructed across the views." }, { "criterionId": "assembly_count", "A": "pass", "B": "pass", "evidenceA": "Four legs and four perimeter stretchers are distinguishable across the hero and underside views.", "evidenceB": "The combined views establish exactly four legs and four perimeter stretchers." }, { "criterionId": "finish", "A": "fail", "B": "fail", "evidenceA": "The seat has stepped corner cutouts rather than a rounded rectangular perimeter; repetitive wavy grain dominates its surfaces, and side hardware appears raised.", "evidenceB": "Rounded edges and smooth surfaces remain coherent across views, but the conspicuous domed side fasteners do not read as inset hardware." } ], "majorDefects": { "A": [ "Stepped, notched seat silhouette misses the requested rounded rectangle.", "Strong repeating wood texture looks artificial across the seat, rails and underside.", "Raised side fasteners weaken the restrained inset-hardware treatment." ], "B": [ "Side fasteners visibly project instead of sitting inset.", "Nearly uniform tan surfaces provide weak wood-grain readability." ] }, "rationale": "B wins through its clean rounded rectangular seat, controlled edge refinement and more finished silhouette. Both show connected assemblies and preserved openings, but A's notched seat and repetitive grain substantially reduce fidelity and polish. Absolute height cannot be verified from these images." } }, { "mapping": { "A": "current", "B": "vanilla" }, "decodedWinner": "current", "scoresByMode": { "current": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 6.5, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 6, "silhouetteAndProportion": 6.5, "constructionPlausibility": 7, "craftsmanshipAndDetail": 5.5, "materialsAndReadability": 6, "surfaceFinishAndShading": 6, "materialTextureQuality": 4, "lightingAndPresentation": 8, "finalStageCompleteness": 5.5, "multiviewConsistency": 8 } }, "result": { "winner": "A", "confidence": 0.94, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 6.5, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 9 }, "B": { "taskFidelity": 6, "silhouetteAndProportion": 6.5, "constructionPlausibility": 7, "craftsmanshipAndDetail": 5.5, "materialsAndReadability": 6, "surfaceFinishAndShading": 6, "materialTextureQuality": 4, "lightingAndPresentation": 8, "finalStageCompleteness": 5.5, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "seat_support", "A": "pass", "B": "pass", "evidenceA": "Leg tops contact the seat underside; underside and elevation views show no open separation or upper-surface penetration.", "evidenceB": "Visible leg tops meet the underside at the corners without a discernible gap or penetration through the seat top." }, { "criterionId": "stretcher_joints", "A": "pass", "B": "pass", "evidenceA": "All four rails terminate against the legs, with contact visible in the perspective and underside views.", "evidenceB": "The four rails visibly connect to their legs; no detached or exposed protruding rail ends are apparent." }, { "criterionId": "ground_support", "A": "pass", "B": "pass", "evidenceA": "Feet align with the floor across elevations, without visible hovering or burial.", "evidenceB": "Feet terminate consistently at the floor in the opposing elevations." }, { "criterionId": "negative_space", "A": "pass", "B": "pass", "evidenceA": "Under-seat openings and the stretcher-ring interior remain clear of crossing geometry.", "evidenceB": "Openings between the legs and inside the ring remain unobstructed." }, { "criterionId": "assembly_count", "A": "pass", "B": "pass", "evidenceA": "Exactly four legs and four stretchers are identifiable across the perspective and underside views.", "evidenceB": "Exactly four legs and four stretchers are identifiable across the supplied views." }, { "criterionId": "finish", "A": "pass", "B": "fail", "evidenceA": "Rounded seat edges, softened frame edges and restrained hardware remain coherent across opposing and underside views.", "evidenceB": "Repetitive rippled grain, stepped seat corners and visibly projecting side hardware undermine the requested refined finish." } ], "majorDefects": { "A": [ "Wood surfaces are unusually uniform, with little convincing grain detail." ], "B": [ "Seat has a stepped, cross-like perimeter instead of a rounded rectangular outline.", "Strong repetitive grain appears artificial and poorly scaled across different members.", "Several side fasteners project visibly rather than reading as restrained inset hardware." ] }, "rationale": "A delivers the requested seat silhouette and a more coherent polished finish. Both show connected frames and preserved openings, but B's stepped seat, conspicuous hardware and repetitive texture substantially reduce fidelity and refinement." } } ] }, "id": "joinery_stool--claude-sonnet-5-5--r01", "repetition": 1, "cohort": "October 2026 spatial pilot", "limitation": "One generation per condition; two counterbalanced Astra judgments of the same images. Vanilla runs were collected later and paired with the existing frozen plugin runs; generation order was not counterbalanced. With plugin means pinned skill workflows and bundled scripts, not an MCP ablation. Concurrent times are not a latency benchmark. Structural scores are proxies, not visual quality or proof of contact.", "judgeCount": 2 }, { "task": "task_lamp_clearance_holdout", "taskTitle": "Articulated task-lamp clearance holdout", "model": "claude-sonnet-5-5", "modelTitle": "Sonnet 5.5", "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 147, "triangles": 28146, "images": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/hd/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/hd/bottom.png" }, "imageHashes": { "perspective": "424bbcbc97529022c2f1d18e9ef68b414c846dbda8290d5530a78b1b7457f56c", "front": "45197afac8b779f7af0edef551c3696bb47b5c55f4f88d52479fc01592b14069", "back": "0a89f5b99db67b2e75c1664a306c5e0e491fdeba484e0baaeadfb9880d2cfafd", "left": "81a7b1ef294693a307a7324d98037e1322c64665a6a3b3c1a332c7b8cbe21e29", "right": "6d9ca765bab522eb683d940a3cd841e2fb100ea0178a2458d1155bc4aea010c9", "top": "0a2f77a66fe4a9c50f997d39077deefe8bd577dd431fb805adf4a162aaaa8dca", "bottom": "33d088a0a343d666dd62d259b53af203c2f70119ac2f9f66debea9823c87f68a" }, "failedChecks": [], "artifactHashes": { "create_asset.py": "fcd6639d11ce12c804cb9b3c54dd9f0add67d415f1dfbd64df2b4f2a34f58848", "asset.blend": "ca5919b77b8f042f0484aecd1439b2983e1fe00fc967a896a6d69df251750498", "asset.glb": "236bc6e0de0ef5b298b7b21dff8b847ae0a47ab5dba387c426a5d6c57ec620eb" }, "generationTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "correctedTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "agentVersion": "2.1.287 (Claude Code)", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": null, "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "baseline", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "ca5919b77b8f042f0484aecd1439b2983e1fe00fc967a896a6d69df251750498", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/original/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/vanilla/original/bottom.png" }, "originalImageHashes": { "perspective": "4048532b3452e52a72797af840e9b69375cef302cde56fd1bfdc9cea530cc9f3", "front": "a31a89b64a6a4880e9a0bbc382925ac9e419d00e9e27a02de82f03924b40238f", "back": "0c41fd23c879dad5f631736adf4f2fd01d5a5abc48220376321970f2e990b006", "left": "909d033f214584a57d8080d123abbf00aa3925bf1bb28299f6c360612eb60871", "right": "f3ea5363b215488abb43e6b984e49550cbb11e2dff7d251a2d7353b80cd92fa2", "top": "19bf3fc00f7f3d65bf9c272dd16c906d17144086b8640616fff328ccafa022e2", "bottom": "56bd6230e5a218eb7c88e7f39947bce760da02961863e97708968bab72dbab75" } }, "plugin": { "score": 96, "rawScore": 96, "hardGate": false, "rawHardGate": false, "seconds": 196, "triangles": 27464, "images": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/hd/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/hd/bottom.png" }, "imageHashes": { "perspective": "657821551c4ff2c23cc4905d0a8eb0cbc03bb3ed4e603d1184a6d30353e30464", "front": "70e8b885e59761f90848fa7b231802c36e71494b02595cdabaf84242140a6c3e", "back": "e2260f298a40b1e60e8f46bcd4e7596a29ec0ea8377209f06a0785c293844d79", "left": "a1f4f2b1f101d17e5faa826ffc6c625ce8e051de28de1ee4f11b6da42df23415", "right": "d721508fbae4b84b9de89300f2a037a970ad126cd2bfa60dc28480fd0fa8e51b", "top": "db684215375c220cac78b0b679aed404b99366f2806015cdc88193e0c4b4a5af", "bottom": "695b91046763c41aa5f42217153bf51b2c22b1058ba1ddaaf52a96b5a38f7f6e" }, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "130 invalid vertices, degenerate faces, or zero-length edges" } ], "artifactHashes": { "create_asset.py": "77c96231a274a0b9a9d404aeee36c1723a0a71b41159fbaa8f96f44798e9e6f3", "asset.blend": "e72d5e000367a3ea4c925cf81fba6374fd97543041b30c206f9f7a96b09c5242", "asset.glb": "c2b6c04e16ccefebe83999e25849e4272625dcd28729559ba34cf9038617797b" }, "generationTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "correctedTaskFingerprint": "bd8f19a899051032660b0822937beaa48d6867deeafe19e72184213f4d16c4f5", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "agentVersion": "2.1.287 (Claude Code)", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "rawScorerVersion": 5, "correctedScorerVersion": 6, "executionMode": "skills", "guidanceHash": null, "preview": { "preset": "gallery-cycles-v1", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "hiddenStagingObjects": [], "sourceSha256": "e72d5e000367a3ea4c925cf81fba6374fd97543041b30c206f9f7a96b09c5242", "sourceModified": false }, "originalImages": { "perspective": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/original/top.png", "bottom": "benchmarks/gallery-matrix/task_lamp_clearance_holdout--claude-sonnet-5-5--r01/plugin/original/bottom.png" }, "originalImageHashes": { "perspective": "aae04e3cb009f35bca660b89dbea23bfc2e73605889ff4e2ee4e2a0e46e35947", "front": "c69c38ab9fa8a83a50b14b8bcce3e3b54e4521ec602c3dcc53e219e762b21bf8", "back": "441b509c3f02782c1409dd382cfefb5832e5bc085d72a28c33b334b4983638d7", "left": "0189ca16809c965ce227c707f97d40aec44ab795eb5b68ee4361ee52d2997db0", "right": "fb52c6446303d8edbb6443153f75dc5d65a9db305cb86ca105d5f08ac34dabb0", "top": "3c572863b3ea6219677a5bf88017e2971199bd1abcaf7a99aabf357b33449163", "bottom": "2b14a5d672c02b23aaa306289f3112c1fc685c3115075a9d13792047f7cbcbe9" } }, "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "note": "One review noted: The plugin result provides stronger visible construction evidence, especially its recessed bulb, shade interior, socket and underside pad. Its larger presentation and clearer material separation make the finished assembly easier to assess. The no-plugin result's obscured underside and indistinct bulb leave important requirements unverified.", "criteria": [ { "id": "joint_continuity", "label": "joint continuity", "question": "Are base-to-arm, elbow, and arm-to-shade connections physically continuous through credible hinge hardware?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "hinge_clearance", "label": "hinge clearance", "question": "Do adjacent arms and hinge cheeks retain readable mechanical clearance without unintended intersections?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 1, "fail": 0, "unclear": 1 } }, { "id": "shade_cavity", "label": "shade cavity", "question": "Is the shade mouth open with a distinct recessed bulb that does not pierce the shell?", "critical": true, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cable_contact", "label": "cable contact", "question": "Does the cable meet its socket and rest on the desk plane without floating or being buried?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "base_contact", "label": "base contact", "question": "Does the weighted base sit flat on the supporting plane?", "critical": true, "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "finish", "label": "finish", "question": "Do enamel, brushed metal, rubber and reverse-side construction details read as a finished product?", "critical": false, "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "rawReview": { "taskId": "task_lamp_clearance_holdout", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 96, "hardGates": { "baseline": true, "candidate": false }, "visualWinnerVotes": [ "current", "current" ], "criterionMajorities": [ { "criterionId": "joint_continuity", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "hinge_clearance", "baselinePasses": 0, "candidatePasses": 1, "judgeCount": 2 }, { "criterionId": "shade_cavity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cable_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "base_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "finish", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "joint_continuity", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "hinge_clearance", "baselinePasses": 0, "candidatePasses": 1, "judgeCount": 2 }, { "criterionId": "shade_cavity", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cable_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "base_contact", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "current" }, "decodedWinner": "current", "scoresByMode": { "vanilla": { "taskFidelity": 6.5, "silhouetteAndProportion": 7, "constructionPlausibility": 6, "craftsmanshipAndDetail": 6, "materialsAndReadability": 6.5, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 6, "lightingAndPresentation": 5.5, "finalStageCompleteness": 6.5, "multiviewConsistency": 7 }, "current": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "result": { "winner": "B", "confidence": 0.87, "scores": { "A": { "taskFidelity": 6.5, "silhouetteAndProportion": 7, "constructionPlausibility": 6, "craftsmanshipAndDetail": 6, "materialsAndReadability": 6.5, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 6, "lightingAndPresentation": 5.5, "finalStageCompleteness": 6.5, "multiviewConsistency": 7 }, "B": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "joint_continuity", "A": "pass", "B": "pass", "evidenceA": "Visible pivot hardware connects the base, paired arms, elbow and shade mount.", "evidenceB": "Base bracket, elbow pins and shade hinge hardware visibly connect the articulated assembly." }, { "criterionId": "hinge_clearance", "A": "unclear", "B": "unclear", "evidenceA": "Arm separation is visible, but the small hinge regions do not resolve cheek-to-arm clearance.", "evidenceB": "Paired rails and hinge hardware are readable, but mating surfaces at the elbow do not clearly establish clearance." }, { "criterionId": "shade_cavity", "A": "unclear", "B": "pass", "evidenceA": "The shade has a light-colored opening, but a distinct recessed bulb cannot be resolved.", "evidenceB": "The frontal opening shows an interior reflector and a distinct bulb recessed within the rim, without visible shell penetration." }, { "criterionId": "cable_contact", "A": "pass", "B": "pass", "evidenceA": "The cable visibly meets the base and follows the tabletop surface in perspective and side views.", "evidenceB": "A visible socket connection leads into a cable that settles onto the supporting plane; side views show the long run resting on it." }, { "criterionId": "base_contact", "A": "pass", "B": "pass", "evidenceA": "The round base's lower edge sits flush against the tabletop in the side views.", "evidenceB": "The base's dark lower layer meets the supporting surface consistently across elevation views." }, { "criterionId": "finish", "A": "unclear", "B": "pass", "evidenceA": "Smooth mint enamel and contrasting hardware are visible, but the tabletop entirely occludes reverse-side construction and rubber detailing.", "evidenceB": "Glossy teal enamel, contrasting metal hardware, dark cable and a distinct underside pad create a coherent finished product; metal brushing is subtle." } ], "majorDefects": { "A": [ "The tabletop completely obscures the underside inspection.", "A distinct recessed bulb is not visibly established.", "Small hinge details leave mechanical clearance unresolved." ], "B": [ "Elbow and hinge-cheek clearances remain insufficiently resolved." ] }, "rationale": "B provides stronger visible construction evidence, especially its recessed bulb, shade interior, socket and underside pad. Its larger presentation and clearer material separation make the finished assembly easier to assess. A's obscured underside and indistinct bulb leave important requirements unverified." } }, { "mapping": { "A": "current", "B": "vanilla" }, "decodedWinner": "current", "scoresByMode": { "current": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 7.5, "finalStageCompleteness": 8, "multiviewConsistency": 8 }, "vanilla": { "taskFidelity": 6.5, "silhouetteAndProportion": 7.5, "constructionPlausibility": 6.5, "craftsmanshipAndDetail": 6.5, "materialsAndReadability": 7, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 6.5, "lightingAndPresentation": 5.5, "finalStageCompleteness": 6.5, "multiviewConsistency": 7 } }, "result": { "winner": "A", "confidence": 0.83, "scores": { "A": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 7.5, "finalStageCompleteness": 8, "multiviewConsistency": 8 }, "B": { "taskFidelity": 6.5, "silhouetteAndProportion": 7.5, "constructionPlausibility": 6.5, "craftsmanshipAndDetail": 6.5, "materialsAndReadability": 7, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 6.5, "lightingAndPresentation": 5.5, "finalStageCompleteness": 6.5, "multiviewConsistency": 7 } }, "criterionResults": [ { "criterionId": "joint_continuity", "A": "pass", "B": "pass", "evidenceA": "Visible pivot hardware connects the base, arm segments and shade bracket.", "evidenceB": "Base pivot, elbow pin and shade pivot visibly connect the articulated components." }, { "criterionId": "hinge_clearance", "A": "pass", "B": "unclear", "evidenceA": "Front and rear views expose separated arm rails; upper hardware shows spaced cheeks around the pivot.", "evidenceB": "Arm rails are separated, but the small elbow and shade connections do not clearly expose their mating clearances." }, { "criterionId": "shade_cavity", "A": "pass", "B": "unclear", "evidenceA": "The forward view shows an open white-lined shade with a distinct bulb recessed inside the rim.", "evidenceB": "A pale shade interior is visible, but a distinct recessed bulb and its separation from the shell are not resolved." }, { "criterionId": "cable_contact", "A": "pass", "B": "pass", "evidenceA": "Cable visibly joins the base fitting and follows the supporting surface in perspective and side views.", "evidenceB": "Cable meets the base and lies along the desktop surface, supported by perspective and side views." }, { "criterionId": "base_contact", "A": "pass", "B": "pass", "evidenceA": "The round base has a flat lower edge seated on the floor without a visible gap.", "evidenceB": "The round base sits flush against the desktop in both side views." }, { "criterionId": "finish", "A": "pass", "B": "unclear", "evidenceA": "Smooth turquoise enamel, bright metal fittings, dark cable and a contrasting underside pad provide finished material separation.", "evidenceB": "Smooth mint surfaces and metal fittings read cleanly, but the retained desktop completely conceals reverse-side construction in the underside tile." } ], "majorDefects": { "A": [], "B": [ "The desktop completely blocks the underside inspection.", "Bulb recession and shell separation are not visibly established.", "Small hinge hardware leaves mechanical clearance unresolved." ] }, "rationale": "A provides stronger visible construction evidence, a readable recessed bulb, clearer hardware separation and an exposed finished underside. B has a coherent smooth silhouette, but its smaller presentation and obstructed underside leave several required relationships unverified." } } ] }, "id": "task_lamp_clearance_holdout--claude-sonnet-5-5--r01", "repetition": 1, "cohort": "October 2026 spatial pilot", "limitation": "One generation per condition; two counterbalanced Astra judgments of the same images. Vanilla runs were collected later and paired with the existing frozen plugin runs; generation order was not counterbalanced. With plugin means pinned skill workflows and bundled scripts, not an MCP ablation. Concurrent times are not a latency benchmark. Structural scores are proxies, not visual quality or proof of contact.", "judgeCount": 2 }, { "id": "tabletop_press--claude-sonnet-5-5--r01", "task": "tabletop_press", "taskTitle": "Tabletop lever press", "model": "claude-sonnet-5-5", "modelTitle": "Sonnet 5.5", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "tabletop_press", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 96.12, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "vanilla", "vanilla" ], "criterionMajorities": [ { "criterionId": "press_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_drive_chain", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_supports", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "press_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_drive_chain", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "press_supports", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 8, "silhouetteAndProportion": 7, "constructionPlausibility": 7, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.5, "lightingAndPresentation": 8.5, "finalStageCompleteness": 7.5, "multiviewConsistency": 9 }, "vanilla": { "taskFidelity": 8.5, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "result": { "winner": "B", "confidence": 0.86, "scores": { "A": { "taskFidelity": 8, "silhouetteAndProportion": 7, "constructionPlausibility": 7, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.5, "lightingAndPresentation": 8.5, "finalStageCompleteness": 7.5, "multiviewConsistency": 9 }, "B": { "taskFidelity": 8.5, "silhouetteAndProportion": 9, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 9, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 9 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "Base, twin uprights, side lever and pivot, exposed rack assembly, ram, die, gold spring, and collection tray are visible.", "evidenceB": "Base, braced upright, overhead lever and pivot, short linkage, ram, die, gold spring, and collection tray are visible." }, { "criterionId": "press_drive_chain", "A": "pass", "B": "pass", "evidenceA": "Lever meets the transverse pivot shaft at the exposed rack drive; the vertical ram aligns with the lower die.", "evidenceB": "Lever pivots above the frame and connects through a short vertical linkage to the ram, which aligns with the die." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "Spring surrounds the ram beneath its guide; die and tray have visible base support, and both uprights terminate on mounting feet.", "evidenceB": "Ram passes through the upper housing, spring is retained around its lower shaft, and die and tray rest on the base; a gusset reinforces the upright." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Painted frame, dark tooling, metallic fasteners, gold spring, and rounded grip provide coherent separation, though the frame remains visually basic.", "evidenceB": "Consistent bevels, smoothly shaded tooling, integrated frame transitions, mounting hardware, and contrasting grip and tray produce a finished industrial appearance." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins through a stronger braced silhouette, clearer lever-to-ram relationship, and more consistently refined surfaces. A satisfies the visible requirements, but its thin rectangular frame and crowded exposed drive assembly feel less resolved." } }, { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.5, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 8.4, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.6, "materialTextureQuality": 8, "lightingAndPresentation": 8.4, "finalStageCompleteness": 8.5, "multiviewConsistency": 8.7 }, "plugin": { "taskFidelity": 8.2, "silhouetteAndProportion": 7.3, "constructionPlausibility": 7.7, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.7, "lightingAndPresentation": 8.3, "finalStageCompleteness": 7.7, "multiviewConsistency": 8.5 } }, "result": { "winner": "A", "confidence": 0.78, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.5, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 8.4, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.6, "materialTextureQuality": 8, "lightingAndPresentation": 8.4, "finalStageCompleteness": 8.5, "multiviewConsistency": 8.7 }, "B": { "taskFidelity": 8.2, "silhouetteAndProportion": 7.3, "constructionPlausibility": 7.7, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.7, "lightingAndPresentation": 8.3, "finalStageCompleteness": 7.7, "multiviewConsistency": 8.5 } }, "criterionResults": [ { "criterionId": "press_parts", "A": "pass", "B": "pass", "evidenceA": "Base, braced upright, top lever and pivot, short linkage, central ram, spring, lower die and gold tray are visible.", "evidenceB": "Base, twin uprights, side lever, transverse pivot shaft, toothed drive, ram, spring, die and separate tray are visible." }, { "criterionId": "press_drive_chain", "A": "pass", "B": "pass", "evidenceA": "Lever connects to a short vertical link above the guided ram; ram and lower die share an axis.", "evidenceB": "Lever connects to the transverse shaft beside the visible toothed drive, with the vertical ram aligned over the die." }, { "criterionId": "press_supports", "A": "pass", "B": "pass", "evidenceA": "Frame is braced to the base; spring surrounds the ram between collars, and die, chute and tray have visible support.", "evidenceB": "Uprights have base feet, the ram passes through a crossmember, the spring has end shoulders, and the tray stands on short legs." }, { "criterionId": "press_finish", "A": "pass", "B": "pass", "evidenceA": "Rounded painted edges, smooth dark ram surfaces, contrasting spring and tray, fasteners and shaped grip create a finished industrial prop.", "evidenceB": "Beveled frame and base, metallic hardware, smooth ram and contrasting grip establish manufactured materials, although the frame remains comparatively plain." } ], "majorDefects": { "A": [], "B": [ "Tall, narrow frame and oversized empty upper opening weaken the compact tabletop proportions.", "Exposed toothed drive is visually crowded, making its engagement less legible.", "Repeated rectangular frame members and simple tray give the assembly a less refined finish." ] }, "rationale": "A wins through its compact, braced silhouette, more integrated collection area and stronger surface refinement. Both visibly satisfy the mechanism requirements, but B's sparse, elongated frame and crowded drive details feel less resolved." } } ] }, "votes": { "baseline": 2, "candidate": 0, "tie": 0 }, "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 83, "triangles": 16516, "images": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "3d79c0dc305970c941b0e2bc385b156994338e75a294dd4e27be0c88e68debf8", "front": "42daaf7e7bef6f574ec3c9576bf94ba6208837e2e8e88856425da51068700cfd", "back": "8bd9b338f6b50f98a96f1f2a59ecb605041144912897f1bf2135c2cca0462935", "left": "99c0fa18975d31b7b9065bd4c661807f9268b14620f74eb6951c91a8354820a6", "right": "053d3d70424633f45d9d01b02e20c27f61c467e694679482861bfca14e73285f", "top": "74a668b0f0452e8bdf1a491c6d6b951e82b76d777e4209d098f7d5c85cf2e417" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "57c77d52238af450b8a46aa9c8c125f14d86e933c515fc1a720dab1a8cff96d5", "front": "b2580aff8c67226c0274e1385c742fd36871e1bdd737bfa0b77db603f21c12ed", "back": "98fa9a2541dc1848675c597642d701ed07fdf17382204af8bf4798f8e5daf861", "left": "3b8b61676f1a7b30899c83e6749f5c88afae8c9ce56a120e91b2d8bbf28a53ed", "right": "e614f54d9101d75220088f65ebb5bd16cbe62bfa2f1943b924eb9528df6a0dec", "top": "c832d00235c5f8cd3f885bc4599e5cdc6775522649519051ba1f646c32eab785" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "b23d0ef798f97a1939f865b2f510379f1620a63bb58e40f2dd0a471dcf3b0429", "front": "a5648a8955c76468b63927018a74bd17271e5748cc48020b6f822b35097ce28f", "back": "f74b752cb2634c4a1f09c27f6f2c2c126f7396943e53f70e3e7ab8db1672a5c1", "left": "b653f56b628b71b60edd57909176e059f8e421c23c2c4cc7629ef60a92ed89dc", "right": "e29de8c50772d8e80122b1bd47e7e3fe617d8ffec4d2d063f8dc67be51510bfb", "top": "61e77cc6a7dd008074302f807d06a02b13305f048382a6d245c05e5d7d4983c7" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "633f81e598231bdc375a3391805476f4afae9b6c24d9e4c44e23715297ffc5e6", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "633f81e598231bdc375a3391805476f4afae9b6c24d9e4c44e23715297ffc5e6" }, "artifactHashes": { "create_asset.py": "01d586761d9572885cb077eb70e079d3e605a79795b35e2a6f3e3b23ec4a56b1", "asset.blend": "633f81e598231bdc375a3391805476f4afae9b6c24d9e4c44e23715297ffc5e6", "asset.glb": "aefb16cb4835049ef4e19ce3ad024af77054f7760a84a3503c9030352e00c2ea" }, "failedChecks": [], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "e147da73ad0c4c84e79c127ae9eb1e37b1a299e26700ddfb96d17df5c0172260", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 5 }, "plugin": { "score": 96.12, "rawScore": 96.12, "hardGate": true, "rawHardGate": true, "seconds": 128, "triangles": 16404, "images": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "aa9ad73b9246ac5c9afa90c113a1d95ada8ac93d169a98fd5de03171219e5ff9", "front": "18187e7fdc9d017e7d30ecfd48ec0772ca878b704eb083bbd55d3092a8a6169a", "back": "857bd38dfc953e64fb07fe31c9b5a361074d15dd607ad00356619f3ae20071a1", "left": "1e863bc1ed8a9e806a2d4fe589f6156e3710436c225fd5b8b6a2a9454d7711b6", "right": "0ea34660cdb785fff3335f7193c3e9b53b5f4e0a3440284a565b10a714d28085", "top": "7f8d2cc9b314289eedf697b4282ab3a2ca977ca076b45d383a75beef4c56f47f" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "204c5065f64ed1111b38275c4a73d8c288f3979517d3e6b92f9fc7714b8bafda", "front": "f6c7560e80abcfe36f4191a6df0b578d8aa88b51f447887b434895e0cb9cbf23", "back": "33bf2285ffbfb708144c431d15b877730c49d85efa640b0b4eeea39cf747641f", "left": "974f634b24193b4d5c031e31fea81b110727131ed1f941a3fb222358a97849c0", "right": "574e4f93540055b9f90bcb52a3f95460b8208ab336321ed3d166d042fde802b7", "top": "71991d8ea6fe6e40ee5066a41d2b6b4253a5ee715352be44c53d85f9c4eb8388" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/tabletop_press--claude-sonnet-5-5--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "c85865f9e12c5d1d3bbfc982c99da66e9cdd4d07edcff5a39092ed674f90997b", "front": "5c99b6170cf5c9ba501b5e4c831e04f6110d36259322ced26713435014e97d18", "back": "67e39e6b4af418e4419e99e1bc82efe28a8c3ab5d3bacecffbc2aa3a0aab96d8", "left": "a3ac68aa56cf8fff4a0be91141802b2e73ad9d4ecb8ca15619c22a6ffd972f0d", "right": "019bef3c9df23770e86b206d0141388e0b75d7a3db06e0906c6bb0e51c7dd837", "top": "25ce89acfbcda991f1b2293b5af58a82b2cf7b36e23bda81a5904d2230d6e95a" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "678612f27efc1fa20d5a8514fbd8ba14e64c126633bfbc94108bddf81dc9a947", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "678612f27efc1fa20d5a8514fbd8ba14e64c126633bfbc94108bddf81dc9a947" }, "artifactHashes": { "create_asset.py": "dd3aa2571094523b6cdf6e0dfde9c721693ba5cd09b6fcb4ba1c22cfc761b2a9", "asset.blend": "678612f27efc1fa20d5a8514fbd8ba14e64c126633bfbc94108bddf81dc9a947", "asset.glb": "7235cd97adf3b8b3863c63642ceeed189a1aea71df8ad1907dd3c19f66b16646" }, "failedChecks": [ { "id": "uv_coverage", "passed": false, "earned": 0.11904761904761904, "possible": 4, "detail": "1/56 meshes have UV layers; minimum ratio 0.60" } ], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "e147da73ad0c4c84e79c127ae9eb1e37b1a299e26700ddfb96d17df5c0172260", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 5 }, "criteria": [ { "id": "press_parts", "label": "press parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_drive_chain", "label": "press drive chain", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_supports", "label": "press supports", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "press_finish", "label": "press finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The no-plugin result wins through a stronger braced silhouette, clearer lever-to-ram relationship, and more consistently refined surfaces. The plugin result satisfies the visible requirements, but its thin rectangular frame and crowded exposed drive assembly feel less resolved." }, { "id": "winch_drawbridge--claude-sonnet-5-5--r01", "task": "winch_drawbridge", "taskTitle": "Animated miniature winch drawbridge", "model": "claude-sonnet-5-5", "modelTitle": "Sonnet 5.5", "repetition": 1, "cohort": "October 2026 matrix expansion", "judgeCount": 2, "limitation": "One generation per condition, medium effort, 12-minute generation limit. Two counterbalanced Astra reviews. Conditions generated concurrently; time is not a latency benchmark. Skills and bundled scripts are pinned; this is not an MCP ablation. This gallery shows still views; reviewers also received sampled animation frames when available. Review framing excludes the same named studio meshes as HD. Original submission views and technical scores are preserved separately; oversized staging still fails the technical checks.", "rawReview": { "taskId": "winch_drawbridge", "repetition": 1, "baselineAutomatedScore": 96, "candidateAutomatedScore": 98.06, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "plugin", "vanilla" ], "criterionMajorities": [ { "criterionId": "bridge_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_hinge", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_cables", "baselinePasses": 2, "candidatePasses": 1, "judgeCount": 2 }, { "criterionId": "bridge_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "bridge_parts", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_hinge", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "bridge_cables", "baselinePasses": 2, "candidatePasses": 1, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 8, "silhouetteAndProportion": 7.5, "constructionPlausibility": 7, "craftsmanshipAndDetail": 6.8, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 8, "materialTextureQuality": 6.5, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 8 }, "plugin": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8.5 } }, "result": { "winner": "B", "confidence": 0.82, "scores": { "A": { "taskFidelity": 8, "silhouetteAndProportion": 7.5, "constructionPlausibility": 7, "craftsmanshipAndDetail": 6.8, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 8, "materialTextureQuality": 6.5, "lightingAndPresentation": 8, "finalStageCompleteness": 7, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8.5 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two towers, planked deck, crossbeam, drum, external crank, hinge hardware and paired cables are visible.", "evidenceB": "All required components are visible, including paired cables, substantial hinge hardware and the drum beneath the crossbeam." }, { "criterionId": "bridge_hinge", "A": "pass", "B": "pass", "evidenceA": "The three poses retain the deck's base at the visible hinge while its free end rises.", "evidenceB": "Lowered, intermediate and raised poses consistently pivot around the visible bottom hinge." }, { "criterionId": "bridge_cables", "A": "pass", "B": "pass", "evidenceA": "Paired cable runs change direction toward the rising deck and remain plausibly attached in the supplied poses, with partial tower occlusion.", "evidenceB": "Paired cables plausibly connect the elevated winch to the deck, shortening as the deck rises; some routing is occluded in the raised pose." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Consistent smooth surfaces, softened edges and distinct wood, masonry and metal finishes; tower detailing is comparatively sparse.", "evidenceB": "Consistent bevels, masonry courses, beam braces and hinge details give the assembly a more finished appearance across views and poses." } ], "majorDefects": { "A": [ "Low drum placement crowds the opening and makes raised-state cable routing less readable.", "Broad plain tower surfaces and unexplained thin red accents weaken finish coherence." ], "B": [ "Deep supports obscure much of the mechanism in side views.", "Materials remain fairly uniform, with limited visible surface texture." ] }, "rationale": "B wins through more substantial construction, clearer elevated winch placement, reinforced joinery and richer surface refinement. A demonstrates the requested poses but looks more sparsely developed. Continuous timing, loop smoothness, flicker and between-frame collisions remain unclear for both." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 7.5, "silhouetteAndProportion": 7, "constructionPlausibility": 7.2, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 7.4, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 6.5, "lightingAndPresentation": 7.8, "finalStageCompleteness": 7.5, "multiviewConsistency": 8 }, "vanilla": { "taskFidelity": 8.7, "silhouetteAndProportion": 8.4, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.1, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 6.8, "lightingAndPresentation": 8.3, "finalStageCompleteness": 8.2, "multiviewConsistency": 8.5 } }, "result": { "winner": "B", "confidence": 0.84, "scores": { "A": { "taskFidelity": 7.5, "silhouetteAndProportion": 7, "constructionPlausibility": 7.2, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 7.4, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 6.5, "lightingAndPresentation": 7.8, "finalStageCompleteness": 7.5, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8.7, "silhouetteAndProportion": 8.4, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.1, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 6.8, "lightingAndPresentation": 8.3, "finalStageCompleteness": 8.2, "multiviewConsistency": 8.5 } }, "criterionResults": [ { "criterionId": "bridge_parts", "A": "pass", "B": "pass", "evidenceA": "Two substantial supports, planked deck, crossbeam, upper drum, exterior crank, hinge hardware and paired cables appear across the views.", "evidenceB": "All required parts are readily identifiable, including the exposed drum, axle bearings, crank, deck hinge and two cables." }, { "criterionId": "bridge_hinge", "A": "pass", "B": "pass", "evidenceA": "The three poses show the deck rising around the same visible lower hinge axis.", "evidenceB": "Lowered, intermediate and raised poses retain the deck's attachment to the visible base hinge." }, { "criterionId": "bridge_cables", "A": "unclear", "B": "pass", "evidenceA": "Both cables are established in the static views, but the deep supports and raised deck conceal enough routing and attachments that both connections cannot be verified throughout the critical poses.", "evidenceB": "Paired cables link the drum region to the deck's outer corners, changing direction plausibly with the displayed deck positions." }, { "criterionId": "bridge_finish", "A": "pass", "B": "pass", "evidenceA": "Softened masonry edges, consistent plank divisions and distinct metal hardware remain coherent across views and poses, although broad surfaces are plain.", "evidenceB": "Rounded drum surfaces, beveled supports, deck edging and contrasting metal fittings form a consistent finished miniature across the supplied images." } ], "majorDefects": { "A": [ "Deep slab-like supports dominate the silhouette and obscure the winch and cable relationships.", "Cable continuity is insufficiently exposed in the intermediate and raised animation views." ], "B": [ "Large support faces and nearly uniform wood surfaces lack convincing material texture." ] }, "rationale": "B offers clearer mechanical relationships, a better-proportioned projecting deck and more readable drum, bearings and cable routing. A has appealing masonry detailing and smooth finish, but its bulky supports hide critical construction. Neither set of stills establishes continuous timing, loop smoothness, flicker absence or collision-free motion between frames." } } ] }, "votes": { "baseline": 1, "candidate": 1, "tie": 0 }, "vanilla": { "score": 96, "rawScore": 96, "hardGate": true, "rawHardGate": true, "seconds": 116, "triangles": 6536, "images": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/hd/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/hd/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/hd/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/hd/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/hd/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/hd/top.png" }, "imageHashes": { "perspective": "f97d0233b184a752d91000a9cac02d8ad59e4c692e43a7f360abdf00852a9ef3", "front": "3c15213702505270530d1716a142c5db8007db388aa1bcb0c8e1231c1bb7964e", "back": "e3b3f3b921e65ea53c2ffe81cae47d622722c40e922503ff1624f86d347514c2", "left": "b73dd6f8b920021b69a4be6c41f1e4e716c67e315d6463ef21ce03a739586126", "right": "c8601e20b9d7056701b3cecb31db07e395b34d4e1dc931601fcef7fe767bcc99", "top": "e8476614b2889642113f0af034c7c605ea773c7ea7c7ebadf5dfbc54b8f94377" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/original/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/original/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/original/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/original/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/original/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/original/top.png" }, "originalImageHashes": { "perspective": "dbf8ba54ab1076a3f36a2092b861ba2fc5a36ed4b73b4c7ae128e73dcb69f21b", "front": "df7c034d58939a9e18d722399f9a23f813d053bfb382207d6b0543105af99726", "back": "cc179671e84b657e219e7f7ee3d856788bae6c5f283898fab7256e6c165398ef", "left": "690bc4b015851f41363917cff7f84f15291a6ae1feec3ce6229551eed4ffc73f", "right": "e88eb140abe71e7809fe13f4b79cafb44c9812b4b2a575f28e4e333564d3454b", "top": "d8a1a5396e6bbaf14248a8744819b473c6e5ed0349750e1a6258f137916e0c25" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/raw/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/raw/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/raw/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/raw/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/raw/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/vanilla/raw/top.png" }, "rawImageHashes": { "perspective": "a178b16b0d92bbf8d52edd9d4ce107295504fabd552a8563a698cb0e164a5bf6", "front": "36fe528eaa96c942d3a96846335565fa37cca60e98fe4cc12b85cb46e5c824d9", "back": "bd582e72305c5cde2cc4f859c71ee7936660634feb6cab58df879e860171758e", "left": "b2dedcb106faabe836ca912e97636f8960992bf18e6dc587155a46abb0ca8174", "right": "389ebda91e085fe73ad4252f651af03d8e61352921000d3ccee8aa4128dc6b51", "top": "99c43456c10000980d3efccc36ef9f91ac2552719fb1bddf79dbd1fcf55be5a9" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "e8bf08a4347f454824684ed4c078769683589804b6612286ea51caf8fd8eb7ec", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "e8bf08a4347f454824684ed4c078769683589804b6612286ea51caf8fd8eb7ec" }, "artifactHashes": { "create_asset.py": "31702eb5820aaf704ed96b08e0a868b3ad5685826d955554f9c081aa06c66eb8", "asset.blend": "e8bf08a4347f454824684ed4c078769683589804b6612286ea51caf8fd8eb7ec", "asset.glb": "edc572be963ff99fe97fedccb9145e7b8acf3a1f0ea7381174c0857bcd95d095" }, "failedChecks": [ { "id": "finish_shading_profile", "passed": false, "earned": 0, "possible": 4, "detail": "0.0% smooth polygons for polished_smooth" } ], "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "1a47e1548eb2cabc9674cee1f96b10604b5692d3aa3d237101240d3298d40661", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 5 }, "plugin": { "score": 98.06, "rawScore": 98.06, "hardGate": true, "rawHardGate": true, "seconds": 167, "triangles": 7036, "images": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/hd/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/hd/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/hd/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/hd/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/hd/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/hd/top.png" }, "imageHashes": { "perspective": "bb44adb50eb180af68f1d396c5b353b8c1267150b76210c170354f31b17b1fd4", "front": "2a73810e0d565a0aa184429913eb3aceb5a976aeccbf186879336e5f14b6399b", "back": "ad522253e9d41eed03add37aea687537834b700013b0d267498260d51391f124", "left": "fdba7a079ca3c17ef582eef865b1fb30c3095cd74764514d221346a6a65cf529", "right": "016ed57b49dd897e9a7508ef75d65b4d544ab581975a9b1139b5e29e1335a35c", "top": "23c44ecbbe4ff153b7214debb3b6c0a4a709bb96ab5716651f015b97e5d2f939" }, "originalImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "ba81ea1e38fc2775b0d36527425ff189cda6f38ef9ae589a87f3916831c5cdb3", "front": "9294823d6b55b8e5c91ab57697f1f910afa85c443bc14b3deeb3f01d9745fdc6", "back": "49b741254a262b607348c5f677140a923da053b0896d2f5e15bfc4dd88993a42", "left": "11b988aeafbe506da80af9ab9a73b479c610d6a84ac393d84e3260bf972ede1c", "right": "47409c23e338ebc20bfc419832964811cbbc25d90f8d9f57c9c3d92b609998f2", "top": "9967a0015c4f65db2f308ac1c238978cd67fd8d65878da861c52631a3c38e535" }, "rawImages": { "perspective": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/raw/perspective.png", "front": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/raw/front.png", "back": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/raw/back.png", "left": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/raw/left.png", "right": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/raw/right.png", "top": "benchmarks/gallery-matrix/winch_drawbridge--claude-sonnet-5-5--r01/plugin/raw/top.png" }, "rawImageHashes": { "perspective": "5221c98662a6b51f0689e3edfbe082bce8ecf47eb403cb548c1c9c19179af8b2", "front": "ec6014056e7f8e10f5d920c2977a23f854e27f72ed7ab4817c382154946e5758", "back": "fe055b0964b6bb7802925b6ff645a65038058b85ed1f354f1981c55055944154", "left": "66d3720e2be8eba748bdad0db69d1e62abbf8c7f2cb1ff5e9f56dca1eb972c2b", "right": "9fe3c5fdf30ac4bdff8efdcb9e4627aa90eea90b51fe32246b2ac350bb00c051", "top": "54a940fec1d5f447d7b3902ffd4edf2f3a33dfbb3eddf896365b04ea3f9e5096" }, "preview": { "preset": "gallery-cycles-v1", "sourceSha256": "642883ac3abbe3a82865d3e72faaf48f45495e80c63e000d1d1fdb3d92aaab92", "resolution": 1536, "engine": "CYCLES", "samples": 64, "denoiser": "OPENIMAGEDENOISE", "device": "OPTIX", "views": [ "perspective", "front", "back", "left", "right", "top" ], "hiddenStagingObjects": [], "sourceModified": false, "blenderVersion": "5.2.2 LTS", "purpose": "Display preview only. Historical scores and reviews refer to original evidence." }, "reviewFraming": { "policy": "neutral-staging-excluded-v1", "hiddenStagingObjects": [], "sourceSha256": "642883ac3abbe3a82865d3e72faaf48f45495e80c63e000d1d1fdb3d92aaab92" }, "artifactHashes": { "create_asset.py": "fd969fca2c2d1c5c006c5f4c392a3a8d0bc890a6f04cc75bb04290fce7c0addf", "asset.blend": "642883ac3abbe3a82865d3e72faaf48f45495e80c63e000d1d1fdb3d92aaab92", "asset.glb": "7c7f668f11103fed9979cecf55507bdf7a36b98bcb6c8a3784439c4e3dd5f92a" }, "failedChecks": [ { "id": "finish_shading_profile", "passed": false, "earned": 2.0553846153846154, "possible": 4, "detail": "10.3% smooth polygons for polished_smooth" } ], "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "fee033ab8962b7ea2160eacdb44e4d1584ed8e45fb7d69fd0fa9534181a3deaa", "evaluatorFingerprint": "2bfe2b06f6c16d250c28726fb5e8bdf0a8c9c67875db63b40d0bcea6a0ea4224", "generationTaskFingerprint": "1a47e1548eb2cabc9674cee1f96b10604b5692d3aa3d237101240d3298d40661", "agentVersion": "2.1.288 (Claude Code)", "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 5 }, "criteria": [ { "id": "bridge_parts", "label": "bridge parts", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "bridge_hinge", "label": "bridge hinge", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "bridge_cables", "label": "bridge cables", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 1, "fail": 0, "unclear": 1 } }, { "id": "bridge_finish", "label": "bridge finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "One review noted: The plugin result wins through more substantial construction, clearer elevated winch placement, reinforced joinery and richer surface refinement. The no-plugin result demonstrates the requested poses but looks more sparsely developed. Continuous timing, loop smoothness, flicker and between-frame collisions remain unclear for both." }, { "id": "decorated_reading_room--gpt-6-astra--r01", "task": "decorated_reading_room", "taskTitle": "Complete decorated reading room", "model": "gpt-6-astra", "modelTitle": "GPT 6 Astra", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 2, "rawReview": { "taskId": "decorated_reading_room", "repetition": 1, "baselineAutomatedScore": 96, "candidateAutomatedScore": 98, "hardGates": { "baseline": false, "candidate": false }, "visualWinnerVotes": [ "plugin", "vanilla" ], "criterionMajorities": [ { "criterionId": "room_zones", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_composition", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_secondary_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_materials", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_lighting", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "room_zones", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_composition", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_secondary_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_materials", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8, "materialsAndReadability": 7.7, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.3, "multiviewConsistency": 8.5 }, "plugin": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.2, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 8.5, "lightingAndPresentation": 8.2, "finalStageCompleteness": 8.7, "multiviewConsistency": 8.5 } }, "result": { "winner": "B", "confidence": 0.76, "scores": { "A": { "taskFidelity": 8.6, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8, "materialsAndReadability": 7.7, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.3, "multiviewConsistency": 8.5 }, "B": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.2, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.4, "materialTextureQuality": 8.5, "lightingAndPresentation": 8.2, "finalStageCompleteness": 8.7, "multiviewConsistency": 8.5 } }, "criterionResults": [ { "criterionId": "room_zones", "A": "pass", "B": "pass", "evidenceA": "Upholstered reading chair, side table, floor lamp and rug complement a furnished desk with chair, lamp and reading materials.", "evidenceB": "Reading chair, side table, floor lamp and rug form a complete nook; the desk includes seating, lighting and an open book." }, { "criterionId": "room_composition", "A": "pass", "B": "pass", "evidenceA": "Wall-mounted shelving anchors the desk, while grouped artwork and the sideboard balance an open central walking route.", "evidenceB": "Rugs distinguish activity zones; artwork, plants and shelving establish focal clusters around a clear central floor." }, { "criterionId": "room_secondary_finish", "A": "pass", "B": "pass", "evidenceA": "Reverse wall includes framed prints, a paneled door, hooks and a woven-front cabinet; shelves contain arranged books and ceramics.", "evidenceB": "Continuous wall paneling, framed botanical art, a finished cabinet and populated shelving carry detail into secondary areas." }, { "criterionId": "room_support", "A": "pass", "B": "pass", "evidenceA": "Visible furniture legs meet the floor; books, vessels and lamps sit on their corresponding surfaces without obvious major gaps.", "evidenceB": "Desk legs, chair frames and cabinet supports are readable; the close-up shows the lamp, pot and books seated on the desktop." }, { "criterionId": "room_materials", "A": "pass", "B": "pass", "evidenceA": "Warm wood, pale upholstery, muted plaster, woven cabinet panels and gold-toned lamp hardware separate clearly, although wood and fabric textures are subdued.", "evidenceB": "Directional wood grain, pale fabric, painted wall panels, matte plaster and brass-toned hardware provide stronger material differentiation." }, { "criterionId": "room_lighting", "A": "pass", "B": "pass", "evidenceA": "Soft warm illumination preserves shelf contents, curtain folds and furniture contact shadows across the supplied views.", "evidenceB": "Warm daylight reveals wood texture and furniture contacts while keeping the reverse wall readable; ceiling highlights are slightly distracting." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins narrowly through more convincing wood texture, richer architectural treatment and better resolved desktop details. A has particularly calm composition and smooth lighting, but its broad wood and fabric surfaces appear more uniform and less materially developed." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 8.5, "silhouetteAndProportion": 7.8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 7.7, "surfaceFinishAndShading": 7.8, "materialTextureQuality": 7.3, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8.2 }, "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.9, "multiviewConsistency": 8.7 } }, "result": { "winner": "B", "confidence": 0.85, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 7.8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 7.7, "surfaceFinishAndShading": 7.8, "materialTextureQuality": 7.3, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8.2 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 8.7, "constructionPlausibility": 8.6, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.6, "finalStageCompleteness": 8.9, "multiviewConsistency": 8.7 } }, "criterionResults": [ { "criterionId": "room_zones", "A": "pass", "B": "pass", "evidenceA": "Reading chair, side table, rug and floor lamp accompany a furnished desk and chair.", "evidenceB": "Upholstered reading nook and equipped writing desk both appear complete and usable." }, { "criterionId": "room_composition", "A": "pass", "B": "pass", "evidenceA": "Rugs define the nook, while shelving and artwork organize the desk and cabinet walls; central circulation remains open.", "evidenceB": "Shelving anchors the desk, the window anchors the nook, and restrained reverse-wall decoration preserves a clear walking route." }, { "criterionId": "room_secondary_finish", "A": "pass", "B": "pass", "evidenceA": "Secondary walls include panel trim, framed botanical art, a decorated cabinet and populated shelving.", "evidenceB": "Cane cabinet fronts, paneled door, hooks with hanging bag, radiator and detailed shelf contents finish the secondary areas." }, { "criterionId": "room_support", "A": "pass", "B": "pass", "evidenceA": "Visible furniture legs, lamp bases, plant pots and shelf contents meet their supporting surfaces without obvious major gaps.", "evidenceB": "Desk and cabinet legs, pedestal table, lamp bases and books show plausible contact; no major unsupported objects are apparent." }, { "criterionId": "room_materials", "A": "pass", "B": "pass", "evidenceA": "Wood grain, pale fabric, muted plaster and golden lamp hardware separate clearly, although wood grain is conspicuous and fabric relatively plain.", "evidenceB": "Warm wood, upholstery, curtains, woven cabinet panels and basket distinguish materials coherently; brass hardware reads subtly." }, { "criterionId": "room_lighting", "A": "pass", "B": "pass", "evidenceA": "Warm diffuse illumination preserves wall colors, wood detail and contact shadows across the shown zones.", "evidenceB": "Soft warm illumination maintains readable shelf recesses, furniture contacts and secondary-wall detail without harsh highlights." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins through more refined upholstery, softened furniture edges, richer shelf detailing and convincingly finished secondary elements, especially the woven cabinet and basket. A satisfies the room requirements but has simpler furnishing details, plainer fabrics and more conspicuous wood texture." } } ] }, "resourcePolicy": { "concurrency": 4, "cpuHardCapPercent": null }, "sharedResourceGuidanceHash": null, "limitation": "One generation per condition, medium effort, 30-minute generation limit. Two counterbalanced Astra reviews. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 1, "candidate": 1, "tie": 0 }, "vanilla": { "score": 96, "rawScore": 96, "hardGate": false, "rawHardGate": false, "seconds": 1590, "triangles": 464740, "images": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "82ce13aa23a0ebcfb97b636c71f7aaecc5a32329daf8850bc8c5dc876e6c28b5", "reverse": "13b7b222d665b125f078c8ff8351d149adfa802907c9e6bf3cf9843a13a79c05", "detail": "b7761242230355e0f096b59fb726255b35ba18f018e62f0c2ed63d8552f9eaa2" }, "imageDimensions": { "hero": [ 1536, 1229 ], "reverse": [ 1536, 1229 ], "detail": [ 1536, 1229 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "74f8ec923d0da8a6153dda74cea72544c1a11a59f54e061ff4b162cadccce9ae", "reverse": "a2608fbce5e24589541329485a5a5a8a3ecc25ec815dcc32295e29023e115fca", "detail": "170b87521df8aea87cdf415a750d58d3fc8c9fa22a4bf22d4a36ec599f15a606" }, "originalImageDimensions": { "hero": [ 512, 410 ], "reverse": [ 512, 410 ], "detail": [ 512, 410 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--gpt-6-astra--vanilla\\decorated_reading_room-r01\\asset.blend", "sourceSha256": "7a4afabcdacb245bd7ea0913ac03eeaa6abba351e384b69b0ae54753a983d72f", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 20, "type": "PERSP", "location": [ 2.6700000762939453, -2.259999990463257, 1.7400000095367432 ], "clipStart": 0.05000000074505806, "clipEnd": 100 }, { "name": "SceneReverse", "lens": 22, "type": "PERSP", "location": [ -2.549999952316284, 0.44999998807907104, 1.75 ], "clipStart": 0.05000000074505806, "clipEnd": 100 }, { "name": "SceneDetail", "lens": 39, "type": "PERSP", "location": [ 0, -0.30000001192092896, 1.6399999856948853 ], "clipStart": 0.05000000074505806, "clipEnd": 100 } ], "lights": [ { "name": "Daylight | broad west window", "type": "AREA", "energy": 280, "color": [ 1, 0.8799999952316284, 0.7200000286102295 ], "hidden": false }, { "name": "Sun | late afternoon", "type": "SUN", "energy": 1.350000023841858, "color": [ 1, 0.7699999809265137, 0.47999998927116394 ], "hidden": false }, { "name": "Reading lamp | warm bulb", "type": "POINT", "energy": 48, "color": [ 1, 0.6399999856948853, 0.3100000023841858 ], "hidden": false }, { "name": "Reading lamp | downcast", "type": "AREA", "energy": 22, "color": [ 1, 0.699999988079071, 0.38999998569488525 ], "hidden": false }, { "name": "Desk lamp | warm bulb", "type": "POINT", "energy": 12, "color": [ 1, 0.7300000190734863, 0.44999998807907104 ], "hidden": false }, { "name": "Interior | ceiling bounce", "type": "AREA", "energy": 65, "color": [ 1, 0.9100000262260437, 0.800000011920929 ], "hidden": false } ], "volumeMaterials": [], "world": "Sky | soft blue ambient", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": 0, "gamma": 1 }, "resolution": [ 1920, 1536, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6-astra--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 8.985, "sha256": "82ce13aa23a0ebcfb97b636c71f7aaecc5a32329daf8850bc8c5dc876e6c28b5" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6-astra--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 8.909, "sha256": "13b7b222d665b125f078c8ff8351d149adfa802907c9e6bf3cf9843a13a79c05" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6-astra--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 7.763, "sha256": "b7761242230355e0f096b59fb726255b35ba18f018e62f0c2ed63d8552f9eaa2" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1229 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "2518 invalid vertices, degenerate faces, or zero-length edges" } ], "rawFailedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "2518 invalid vertices, degenerate faces, or zero-length edges" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--gpt-6-astra--vanilla\\decorated_reading_room-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "d14456a716f107c8fe0873dc229f5b3f494c7f54b5270be08f5b3ba0329fd497", "asset.blend": "7a4afabcdacb245bd7ea0913ac03eeaa6abba351e384b69b0ae54753a983d72f" }, "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2206278b2ba4509fe6848dc73cb77a769b4d4c07cb6845c90974f8af8705768f", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6-astra", "modelsObserved": null, "rawScorerVersion": 7 }, "plugin": { "score": 100, "rawScore": 98, "hardGate": true, "rawHardGate": false, "seconds": 1431, "triangles": 331036, "images": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "ebceb2280bd0b5b1fb02b3a6e9592141d9d02e4b2f1a30f3923f701b1be04442", "reverse": "7cb0461fbcc81829afae3276a2a7c9f6fbb50e985f34632915bfeba4d8030b8f", "detail": "4fbf43054da36cc6b3a458209ee2494094e03bbad38cc419d3be2aa81298548e" }, "imageDimensions": { "hero": [ 1536, 1126 ], "reverse": [ 1536, 1126 ], "detail": [ 1536, 1126 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/plugin/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/plugin/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6-astra--r01/plugin/original/detail.png" }, "originalImageHashes": { "hero": "3eb90c3e4f795ed764defa280b41fc4d46405e286fa43263e99ffe17a7fe289e", "reverse": "127ef87db99039272de4a0e3d5c1c9f56e2c0f5cc9b9ff536d3265ef52aa63cd", "detail": "0db16f79d9d6a292dd5217585a355324f562aaa395dbec7d13518753695b4eb1" }, "originalImageDimensions": { "hero": [ 512, 375 ], "reverse": [ 512, 375 ], "detail": [ 512, 375 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--gpt-6-astra--plugin\\decorated_reading_room-r01\\asset.blend", "sourceSha256": "9d74eb5db7a8998fa1706faea270821db001b5e24fec7dbd85654d871c99fb82", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 23, "type": "PERSP", "location": [ 2.700000047683716, -2.2200000286102295, 1.6200000047683716 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneReverse", "lens": 23, "type": "PERSP", "location": [ -2.6500000953674316, 0.949999988079071, 1.7999999523162842 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneDetail", "lens": 43, "type": "PERSP", "location": [ -0.4000000059604645, -0.10000000149011612, 1.649999976158142 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 } ], "lights": [ { "name": "Window broad daylight", "type": "AREA", "energy": 420, "color": [ 1, 0.8500000238418579, 0.6499999761581421 ], "hidden": false }, { "name": "Soft sky window", "type": "AREA", "energy": 220, "color": [ 0.6800000071525574, 0.800000011920929, 1 ], "hidden": false }, { "name": "Interior bounced daylight", "type": "AREA", "energy": 95, "color": [ 1, 0.8700000047683716, 0.6800000071525574 ], "hidden": false }, { "name": "Late afternoon sun", "type": "SUN", "energy": 1.4500000476837158, "color": [ 1, 0.7599999904632568, 0.4699999988079071 ], "hidden": false }, { "name": "Reading lamp warm bulb", "type": "POINT", "energy": 48, "color": [ 1, 0.6200000047683716, 0.30000001192092896 ], "hidden": false }, { "name": "Desk lamp warm pool", "type": "AREA", "energy": 12, "color": [ 1, 0.6800000071525574, 0.3700000047683716 ], "hidden": false } ], "volumeMaterials": [], "world": "World", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": -0.3499999940395355, "gamma": 1 }, "resolution": [ 1800, 1320, 100 ], "frame": 1, "images": [ { "name": "oak_veneer_01_diffuse_2k.jpg", "size": [ 2048, 2048 ], "colorSpace": "sRGB", "packed": true, "source": "FILE" }, { "name": "oak_veneer_01_normal_2k.png", "size": [ 2048, 2048 ], "colorSpace": "Non-Color", "packed": true, "source": "FILE" }, { "name": "oak_veneer_01_roughness_2k.png", "size": [ 2048, 2048 ], "colorSpace": "Non-Color", "packed": true, "source": "FILE" }, { "name": "rough_linen_diffuse_2k.jpg", "size": [ 2048, 2052 ], "colorSpace": "sRGB", "packed": true, "source": "FILE" }, { "name": "rough_linen_normal_2k.png", "size": [ 2048, 2052 ], "colorSpace": "Non-Color", "packed": true, "source": "FILE" }, { "name": "rough_linen_roughness_2k.png", "size": [ 2048, 2052 ], "colorSpace": "Non-Color", "packed": true, "source": "FILE" } ], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6-astra--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 8.575, "sha256": "ebceb2280bd0b5b1fb02b3a6e9592141d9d02e4b2f1a30f3923f701b1be04442" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6-astra--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 7.964, "sha256": "7cb0461fbcc81829afae3276a2a7c9f6fbb50e985f34632915bfeba4d8030b8f" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6-astra--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 8.221, "sha256": "4fbf43054da36cc6b3a458209ee2494094e03bbad38cc419d3be2aa81298548e" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1126 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [ { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "30.000m maximum extent; limit 18m" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--gpt-6-astra--plugin\\decorated_reading_room-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "a78d449a9c03c808352dccdbdfed549f13944b7c0acfbd321559ba2339a10bc9", "asset.blend": "9d74eb5db7a8998fa1706faea270821db001b5e24fec7dbd85654d871c99fb82" }, "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2206278b2ba4509fe6848dc73cb77a769b4d4c07cb6845c90974f8af8705768f", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6-astra", "modelsObserved": null, "rawScorerVersion": 7 }, "criteria": [ { "id": "room_zones", "label": "room zones", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_composition", "label": "room composition", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_secondary_finish", "label": "room secondary finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_support", "label": "room support", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_materials", "label": "room materials", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_lighting", "label": "room lighting", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "Review 1: the plugin result wins narrowly through more convincing wood texture, richer architectural treatment and better resolved desktop details. the no-plugin result has particularly calm composition and smooth lighting, but its broad wood and fabric surfaces appear more uniform and less materially developed. Review 2: the no-plugin result wins through more refined upholstery, softened furniture edges, richer shelf detailing and convincingly finished secondary elements, especially the woven cabinet and basket. the plugin result satisfies the room requirements but has simpler furnishing details, plainer fabrics and more conspicuous wood texture." }, { "id": "decorated_reading_room--gpt-6.1-sol--r01", "task": "decorated_reading_room", "taskTitle": "Complete decorated reading room", "model": "gpt-6.1-sol", "modelTitle": "GPT 6.1 Sol", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 2, "rawReview": { "taskId": "decorated_reading_room", "repetition": 1, "baselineAutomatedScore": 96, "candidateAutomatedScore": 93.5, "hardGates": { "baseline": false, "candidate": false }, "visualWinnerVotes": [ "vanilla", "vanilla" ], "criterionMajorities": [ { "criterionId": "room_zones", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_composition", "baselinePasses": 2, "candidatePasses": 1, "judgeCount": 2 }, { "criterionId": "room_secondary_finish", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "room_support", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "room_materials", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "room_lighting", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "room_zones", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_composition", "baselinePasses": 2, "candidatePasses": 1, "judgeCount": 2 }, { "criterionId": "room_secondary_finish", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "room_support", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "room_materials", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 8 }, "plugin": { "taskFidelity": 4, "silhouetteAndProportion": 7, "constructionPlausibility": 4, "craftsmanshipAndDetail": 6, "materialsAndReadability": 5.5, "surfaceFinishAndShading": 6.5, "materialTextureQuality": 5, "lightingAndPresentation": 5, "finalStageCompleteness": 4, "multiviewConsistency": 7 } }, "result": { "winner": "A", "confidence": 0.98, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 8 }, "B": { "taskFidelity": 4, "silhouetteAndProportion": 7, "constructionPlausibility": 4, "craftsmanshipAndDetail": 6, "materialsAndReadability": 5.5, "surfaceFinishAndShading": 6.5, "materialTextureQuality": 5, "lightingAndPresentation": 5, "finalStageCompleteness": 4, "multiviewConsistency": 7 } }, "criterionResults": [ { "criterionId": "room_zones", "A": "pass", "B": "pass", "evidenceA": "Upholstered reading chair, side table and floor lamp complement a furnished desk with chair, task lamp and writing accessories.", "evidenceB": "Reading chair, side table and rug establish a nook; the separate desk has seating, lighting and accessories." }, { "criterionId": "room_composition", "A": "pass", "B": "pass", "evidenceA": "Window anchors the reading nook, shelf clusters frame the desk, and open floor connects these zones to the reverse-wall console and door.", "evidenceB": "Overhead and perspective views show distinct furniture clusters and an open central walking route." }, { "criterionId": "room_secondary_finish", "A": "pass", "B": "fail", "evidenceA": "Reverse wall includes framed botanical art, a decorated fluted console, finished door and continuous architectural trim.", "evidenceB": "Bookcase and accessories have detail, but enclosing walls, curtains and architectural trim are absent; isolated upright panels remain visually unresolved." }, { "criterionId": "room_support", "A": "pass", "B": "fail", "evidenceA": "Visible furniture legs meet the floor, objects sit on tabletops and shelves, and wall shelves show brackets.", "evidenceB": "Shelves, framed artwork and large rectangular panels float without visible walls or other supporting structures, particularly in side views." }, { "criterionId": "room_materials", "A": "pass", "B": "fail", "evidenceA": "Warm wood grain, soft upholstery and curtains, brass-colored lamps and matte plaster walls remain distinguishable.", "evidenceB": "Pale wood and upholstery are distinguishable, but strong washout weakens material separation; plaster enclosure and convincing brass response are not established." }, { "criterionId": "room_lighting", "A": "pass", "B": "fail", "evidenceA": "Warm interior illumination preserves wood detail, soft contact shadows and readable desk and reverse-wall decoration.", "evidenceB": "Studio lighting against a dark background provides ground shadows but no visible window-light relationship or convincing practical-light contribution; overhead surfaces are washed out." } ], "majorDefects": { "A": [ "Soft, relatively uniform shading limits fine material definition in some secondary surfaces." ], "B": [ "Missing room enclosure, architectural trim and curtains.", "Unsupported floating shelves, artwork and rectangular panels.", "Bright studio illumination washes out material detail.", "Presentation remains a furnished platform rather than a completed interior." ] }, "rationale": "A delivers a cohesive finished interior with supported decoration, architectural detailing and consistent craft across its room views and desk close-up. B has usable furniture zones, but missing architecture and visibly unsupported elements substantially undermine completeness and construction plausibility." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 4, "silhouetteAndProportion": 6, "constructionPlausibility": 3, "craftsmanshipAndDetail": 5, "materialsAndReadability": 4, "surfaceFinishAndShading": 6, "materialTextureQuality": 4, "lightingAndPresentation": 5, "finalStageCompleteness": 3, "multiviewConsistency": 7 }, "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 8, "materialTextureQuality": 8, "lightingAndPresentation": 8, "finalStageCompleteness": 9, "multiviewConsistency": 8 } }, "result": { "winner": "B", "confidence": 0.98, "scores": { "A": { "taskFidelity": 4, "silhouetteAndProportion": 6, "constructionPlausibility": 3, "craftsmanshipAndDetail": 5, "materialsAndReadability": 4, "surfaceFinishAndShading": 6, "materialTextureQuality": 4, "lightingAndPresentation": 5, "finalStageCompleteness": 3, "multiviewConsistency": 7 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 9, "surfaceFinishAndShading": 8, "materialTextureQuality": 8, "lightingAndPresentation": 8, "finalStageCompleteness": 9, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "room_zones", "A": "pass", "B": "pass", "evidenceA": "Armchair, rug and side table establish a reading nook; desk, chair and accessories establish a usable work area.", "evidenceB": "Upholstered reading nook and furnished desk area include seating, lamps, books and usable surfaces." }, { "criterionId": "room_composition", "A": "fail", "B": "pass", "evidenceA": "Furniture clusters occupy an exposed platform with a large empty foreground; unsupported wall elements undermine a complete interior composition.", "evidenceB": "Window and reading chair anchor the main view, desk decoration forms a secondary focus, and open floor connects the zones and door." }, { "criterionId": "room_secondary_finish", "A": "fail", "B": "pass", "evidenceA": "Bookcase and accessories are present, but enclosing walls, architectural trim and curtains are absent; reverse views expose bare panel backs.", "evidenceB": "Reverse wall includes trim, framed art, a detailed door and slatted cabinet; shelves have brackets and arranged books." }, { "criterionId": "room_support", "A": "fail", "B": "pass", "evidenceA": "Shelves and framed decoration visibly float without walls or another supporting structure.", "evidenceB": "Visible furniture legs meet the floor, shelf brackets meet walls, and books, lamps and vessels rest on furniture surfaces." }, { "criterionId": "room_materials", "A": "fail", "B": "pass", "evidenceA": "Pale wood and light upholstery are distinguishable, but brass and plaster lack clear expression and surfaces show little material-specific variation.", "evidenceB": "Wood grain, woven upholstery and curtains, metallic lamp surfaces and softly varied plaster read distinctly at coherent scales." }, { "criterionId": "room_lighting", "A": "fail", "B": "pass", "evidenceA": "Cool studio lighting illuminates the platform; there is no architectural window or convincing warm practical-light treatment.", "evidenceB": "Warm interior illumination and window light preserve wood grain, textile detail, contact shadows and readable reverse-wall furnishings." } ], "majorDefects": { "A": [ "Missing enclosing architecture, trim and curtains.", "Floating shelves and wall decorations have no actual supports.", "Sparse platform presentation and uniform materials leave the room visibly unfinished." ], "B": [ "The supplied views do not expose every furniture underside or all room angles, limiting complete support verification." ] }, "rationale": "B delivers a coherent finished interior with architecture, curtains, supported shelving and convincing material separation. Its reverse wall and desk close-up sustain the main view's craft. A contains recognizable furniture zones but lacks essential architecture and leaves several decorative elements floating." } } ] }, "resourcePolicy": { "cpuHardCapPercent": 20, "priority": "BelowNormal", "concurrency": 1, "threads": 2, "guidanceFile": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-resource-guidance.md", "guidanceHash": "c0083711df10496868e4920ab7482fa23edacc6b264003aee29f18a272cf4735" }, "sharedResourceGuidanceHash": "c0083711df10496868e4920ab7482fa23edacc6b264003aee29f18a272cf4735", "limitation": "One generation per condition, medium effort, 30-minute generation limit. Two counterbalanced Astra reviews. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 2, "candidate": 0, "tie": 0 }, "vanilla": { "score": 96, "rawScore": 96, "hardGate": false, "rawHardGate": false, "seconds": 1541, "triangles": 192836, "images": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "3f8cbc28409e0cf4fae11fbbf85eb27333a6b0d647210377fcc2fcd2034d1857", "reverse": "c708743028d745703d8690455038163398b4d2799530d4ecd0b26fe593c31b22", "detail": "7f67976da9a11788aa889aee40157fc869a0dd46708573449e400671c8bd6a6d" }, "imageDimensions": { "hero": [ 1536, 1152 ], "reverse": [ 1536, 1152 ], "detail": [ 1536, 1152 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "2dfc2e682152ffe2e9ab6fa412a9012a5039a758fca9fb05a5a0ff8ecb70affd", "reverse": "329854283784aa2dd348646726e7d73124a2f8554eb37cda4404aadd24a683c8", "detail": "538848602470fc5d907f90a2854c9bbdd5a29109a872240e756ccf8b5488b203" }, "originalImageDimensions": { "hero": [ 512, 384 ], "reverse": [ 512, 384 ], "detail": [ 512, 384 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\decorated_reading_room--gpt-6.1-sol--vanilla\\decorated_reading_room-r01\\asset.blend", "sourceSha256": "f43cbd941ccdafdefea29e7c650611f74bf9535787cab840ba67579aa3545d14", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "The Quiet Hour | 6 x 5 m reading room", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 21, "type": "PERSP", "location": [ 0.30000001192092896, -2.309999942779541, 1.7599999904632568 ], "clipStart": 0.10000000149011612, "clipEnd": 100 }, { "name": "SceneReverse", "lens": 21, "type": "PERSP", "location": [ 0.5199999809265137, 2.0199999809265137, 1.8600000143051147 ], "clipStart": 0.10000000149011612, "clipEnd": 100 }, { "name": "SceneDetail", "lens": 48, "type": "PERSP", "location": [ 0.10000000149011612, -0.2800000011920929, 2.119999885559082 ], "clipStart": 0.10000000149011612, "clipEnd": 100 } ], "lights": [ { "name": "Sun | late afternoon through north window", "type": "SUN", "energy": 5, "color": [ 1, 0.7599999904632568, 0.49000000953674316 ], "hidden": false }, { "name": "Window | broad soft sky", "type": "AREA", "energy": 160, "color": [ 0.800000011920929, 0.8700000047683716, 1 ], "hidden": false }, { "name": "Interior | gentle ceiling bounce", "type": "AREA", "energy": 45, "color": [ 1, 0.8700000047683716, 0.6800000071525574 ], "hidden": false }, { "name": "Floor lamp | 2700 K practical", "type": "POINT", "energy": 24, "color": [ 1, 0.6100000143051147, 0.28999999165534973 ], "hidden": false }, { "name": "Task lamp | 3000 K practical", "type": "POINT", "energy": 8, "color": [ 1, 0.7300000190734863, 0.4300000071525574 ], "hidden": false } ], "volumeMaterials": [], "world": "Late afternoon | soft exterior", "colorManagement": { "viewTransform": "AgX", "look": "None", "exposure": 0.3499999940395355, "gamma": 1 }, "resolution": [ 1600, 1200, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6.1-sol--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 8.488, "sha256": "3f8cbc28409e0cf4fae11fbbf85eb27333a6b0d647210377fcc2fcd2034d1857" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6.1-sol--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 8.321, "sha256": "c708743028d745703d8690455038163398b4d2799530d4ecd0b26fe593c31b22" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6.1-sol--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 7.863, "sha256": "7f67976da9a11788aa889aee40157fc869a0dd46708573449e400671c8bd6a6d" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1152 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "2340 invalid vertices, degenerate faces, or zero-length edges" } ], "rawFailedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "2340 invalid vertices, degenerate faces, or zero-length edges" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\decorated_reading_room--gpt-6.1-sol--vanilla\\decorated_reading_room-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "3dcdfa251d416ff38e21699b48005741cc31b9ae4259f955f86e880a70ac3cf9", "asset.blend": "f43cbd941ccdafdefea29e7c650611f74bf9535787cab840ba67579aa3545d14" }, "executionMode": "baseline", "guidanceHash": "c0083711df10496868e4920ab7482fa23edacc6b264003aee29f18a272cf4735", "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2206278b2ba4509fe6848dc73cb77a769b4d4c07cb6845c90974f8af8705768f", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 7 }, "plugin": { "score": 93.5, "rawScore": 93.5, "hardGate": false, "rawHardGate": false, "seconds": 1786, "triangles": 280768, "images": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "cb75fb2458c4911ee7a4241f1ad95bab6c52e1c93c09a70e2016b1c159b64076", "reverse": "18f4abd52ddcacd963bc1100b91b68052fb2c1e975c47b5e2fcfe172733f8499", "detail": "2d7c54ae5e097c2a661b6bdf8cea86586a25a1661a2e73aa73b2f090e43611f3" }, "imageDimensions": { "hero": [ 1536, 1152 ], "reverse": [ 1536, 1152 ], "detail": [ 1536, 1152 ] }, "originalImages": { "perspective": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--gpt-6.1-sol--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "69cb1385ccc1c2bb9a010fac09903b7c69d46bd516302e60736f12f946a822b5", "front": "fd469a25404e3fb2092a74f56d381931d50fbf0256f11550a280b43263420ab7", "back": "70b14fe35203b236eb7ab1551393d155d04c8c64b2e80355168d5a03bba04cd6", "left": "402033410c76e423f724ee5cc27645d8e02bc803df5271382cafcce6b05038be", "right": "5db0950dec55aa4fd0c89fdda86b72fd84783f2c1d73458c85bc49fdca90784e", "top": "7572622707c7a91378d244ac81edac9d444fa1e4a1349406e881d33960886e49" }, "originalImageDimensions": { "perspective": [ 512, 512 ], "front": [ 512, 512 ], "back": [ 512, 512 ], "left": [ 512, 512 ], "right": [ 512, 512 ], "top": [ 512, 512 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\decorated_reading_room--gpt-6.1-sol--plugin\\decorated_reading_room-r01\\asset.blend", "sourceSha256": "a39619b23af0877298da52acd455c7bce778c2daee4ba1ca53d8745aaf9d2703", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "TheQuietHour_ReadingRoom", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 22, "type": "PERSP", "location": [ 2.700000047683716, -2.2200000286102295, 1.850000023841858 ], "clipStart": 0.029999999329447746, "clipEnd": 100 }, { "name": "SceneReverse", "lens": 22, "type": "PERSP", "location": [ -2.4800000190734863, 1.2000000476837158, 1.7999999523162842 ], "clipStart": 0.029999999329447746, "clipEnd": 100 }, { "name": "SceneDetail", "lens": 46, "type": "PERSP", "location": [ -0.30000001192092896, -0.3499999940395355, 1.7000000476837158 ], "clipStart": 0.029999999329447746, "clipEnd": 100 } ], "lights": [ { "name": "Reading table lamp practical 2700K", "type": "POINT", "energy": 27.299999237060547, "color": [ 1, 0.6600000262260437, 0.33000001311302185 ], "hidden": false }, { "name": "Desk brass lamp practical 2700K", "type": "POINT", "energy": 21.700000762939453, "color": [ 1, 0.6600000262260437, 0.33000001311302185 ], "hidden": false }, { "name": "Window broad sky fill", "type": "AREA", "energy": 480, "color": [ 0.7799999713897705, 0.8700000047683716, 1 ], "hidden": false }, { "name": "Late afternoon sun", "type": "SUN", "energy": 1.7000000476837158, "color": [ 1, 0.7900000214576721, 0.5199999809265137 ], "hidden": false }, { "name": "Ceiling soft bounced fill", "type": "AREA", "energy": 45, "color": [ 1, 0.8799999952316284, 0.7099999785423279 ], "hidden": false }, { "name": "Entrance wall bounced daylight", "type": "AREA", "energy": 65, "color": [ 0.8199999928474426, 0.8999999761581421, 1 ], "hidden": false } ], "volumeMaterials": [], "world": "Late afternoon sky", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": 0.30000001192092896, "gamma": 1 }, "resolution": [ 1600, 1200, 100 ], "frame": 1, "images": [ { "name": "grey_oak_veneer_01_diffuse_4k.jpg", "size": [ 4096, 4096 ], "colorSpace": "sRGB", "packed": true, "source": "FILE" }, { "name": "grey_oak_veneer_01_normal_4k.png", "size": [ 4096, 4096 ], "colorSpace": "Non-Color", "packed": true, "source": "FILE" }, { "name": "grey_oak_veneer_01_roughness_4k.png", "size": [ 4096, 4096 ], "colorSpace": "Non-Color", "packed": true, "source": "FILE" }, { "name": "rough_linen_diffuse_4k.jpg", "size": [ 4096, 4105 ], "colorSpace": "sRGB", "packed": true, "source": "FILE" }, { "name": "rough_linen_normal_4k.png", "size": [ 4096, 4105 ], "colorSpace": "Non-Color", "packed": true, "source": "FILE" }, { "name": "rough_linen_roughness_4k.png", "size": [ 4096, 4105 ], "colorSpace": "Non-Color", "packed": true, "source": "FILE" } ], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6.1-sol--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 20.746, "sha256": "cb75fb2458c4911ee7a4241f1ad95bab6c52e1c93c09a70e2016b1c159b64076" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6.1-sol--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 21.293, "sha256": "18f4abd52ddcacd963bc1100b91b68052fb2c1e975c47b5e2fcfe172733f8499" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--gpt-6.1-sol--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 20.928, "sha256": "2d7c54ae5e097c2a661b6bdf8cea86586a25a1661a2e73aa73b2f090e43611f3" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1152 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": false, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "authored_render_complete", "passed": false, "earned": 0, "possible": 4, "detail": "All required authored-camera images rendered with dependencies present" }, { "id": "authored_render_dependencies", "passed": false, "earned": 0, "possible": 2.5, "detail": "Authored render has no missing image/library dependencies" }, { "id": "authored_camera_evidence", "passed": false, "earned": 0, "possible": 0, "detail": "All contracted scene cameras must render with available dependencies; this does not certify composition or finish" } ], "rawFailedChecks": [ { "id": "authored_render_complete", "passed": false, "earned": 0, "possible": 4, "detail": "All required authored-camera images rendered with dependencies present" }, { "id": "authored_render_dependencies", "passed": false, "earned": 0, "possible": 2.5, "detail": "Authored render has no missing image/library dependencies" }, { "id": "authored_camera_evidence", "passed": false, "earned": 0, "possible": 0, "detail": "All contracted scene cameras must render with available dependencies; this does not certify composition or finish" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\decorated_reading_room--gpt-6.1-sol--plugin\\decorated_reading_room-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "7708004124ffa94b94cd31d4214b779d687785fb40da3d9a284170bce73b3bc3", "asset.blend": "a39619b23af0877298da52acd455c7bce778c2daee4ba1ca53d8745aaf9d2703" }, "executionMode": "skills", "guidanceHash": "c0083711df10496868e4920ab7482fa23edacc6b264003aee29f18a272cf4735", "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2206278b2ba4509fe6848dc73cb77a769b4d4c07cb6845c90974f8af8705768f", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 7 }, "criteria": [ { "id": "room_zones", "label": "room zones", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_composition", "label": "room composition", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 1, "fail": 1, "unclear": 0 } }, { "id": "room_secondary_finish", "label": "room secondary finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 2, "unclear": 0 } }, { "id": "room_support", "label": "room support", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 2, "unclear": 0 } }, { "id": "room_materials", "label": "room materials", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 2, "unclear": 0 } }, { "id": "room_lighting", "label": "room lighting", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 2, "unclear": 0 } } ], "note": "Review 1: the no-plugin result delivers a cohesive finished interior with supported decoration, architectural detailing and consistent craft across its room views and desk close-up. the plugin result has usable furniture zones, but missing architecture and visibly unsupported elements substantially undermine completeness and construction plausibility. Review 2: the no-plugin result delivers a coherent finished interior with architecture, curtains, supported shelving and convincing material separation. Its reverse wall and desk close-up sustain the main view's craft. the plugin result contains recognizable furniture zones but lacks essential architecture and leaves several decorative elements floating." }, { "id": "decorated_reading_room--claude-opus-5-5--r01", "task": "decorated_reading_room", "taskTitle": "Complete decorated reading room", "model": "claude-opus-5-5", "modelTitle": "Opus 5.5", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 2, "rawReview": { "taskId": "decorated_reading_room", "repetition": 1, "baselineAutomatedScore": 96, "candidateAutomatedScore": 94, "hardGates": { "baseline": false, "candidate": false }, "visualWinnerVotes": [ "vanilla", "plugin" ], "criterionMajorities": [ { "criterionId": "room_zones", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_composition", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_secondary_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_materials", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_lighting", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "room_zones", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_composition", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_secondary_finish", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_materials", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 8.1, "silhouetteAndProportion": 7.8, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 7.4, "materialsAndReadability": 7.2, "surfaceFinishAndShading": 7.6, "materialTextureQuality": 7.1, "lightingAndPresentation": 7.3, "finalStageCompleteness": 7.7, "multiviewConsistency": 8 }, "vanilla": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.2, "constructionPlausibility": 8.3, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.2, "finalStageCompleteness": 8.7, "multiviewConsistency": 8.5 } }, "result": { "winner": "B", "confidence": 0.86, "scores": { "A": { "taskFidelity": 8.1, "silhouetteAndProportion": 7.8, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 7.4, "materialsAndReadability": 7.2, "surfaceFinishAndShading": 7.6, "materialTextureQuality": 7.1, "lightingAndPresentation": 7.3, "finalStageCompleteness": 7.7, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.2, "constructionPlausibility": 8.3, "craftsmanshipAndDetail": 8.6, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.2, "lightingAndPresentation": 8.2, "finalStageCompleteness": 8.7, "multiviewConsistency": 8.5 } }, "criterionResults": [ { "criterionId": "room_zones", "A": "pass", "B": "pass", "evidenceA": "Reading chair, ottoman, side table and lamp form a furnished nook; the desk has a chair, task light and writing accessories.", "evidenceB": "The window nook includes seating, footrest and a lamp table; the furnished desk provides a chair, task lamp and clear writing space." }, { "criterionId": "room_composition", "A": "pass", "B": "pass", "evidenceA": "The rug anchors the nook, with wall-mounted decoration and perimeter furniture preserving an open walking route.", "evidenceB": "Coordinated artwork, shelving and trim organize the perimeter; the rug defines the nook and ample central space connects both zones." }, { "criterionId": "room_secondary_finish", "A": "pass", "B": "pass", "evidenceA": "The reverse wall has a detailed sideboard, framed pictures and accessories; shelves contain books and vessels, though wall treatment is comparatively plain.", "evidenceB": "Continuous architectural molding, a paneled door, articulated sideboard and supported display shelves carry finish into secondary views." }, { "criterionId": "room_support", "A": "pass", "B": "pass", "evidenceA": "Visible furniture legs reach the floor, books rest on shelves, and lamps and accessories meet tabletops without conspicuous gaps.", "evidenceB": "Desk and chair legs, shelf brackets, lamp bases and resting books provide readable support relationships; no major floating parts are apparent." }, { "criterionId": "room_materials", "A": "pass", "B": "pass", "evidenceA": "Wood grain, cream upholstery, curtains, metallic lamps and matte walls are distinguishable, although muted shading reduces their separation.", "evidenceB": "Wood grain, folded fabric, reflective brass and matte plaster read distinctly; the floor reflection is somewhat stronger than the restrained finish elsewhere." }, { "criterionId": "room_lighting", "A": "pass", "B": "pass", "evidenceA": "Window shadows and warm practical pools establish atmosphere, but shelving and the reverse view lose some detail in subdued illumination.", "evidenceB": "Daylight and practical lighting maintain readable secondary furniture and contact shadows, with a bright floor reflection drawing excess attention." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "B wins through more developed architectural trim, stronger material separation and consistently finished secondary furniture. A offers a convincing, intimate reading nook, but its flatter wall decoration and subdued shading make the overall finish less refined. B's oversized empty center and conspicuous floor glare slightly weaken its presentation." } }, { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 8.1, "silhouetteAndProportion": 7.3, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 7.4, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 7.1, "lightingAndPresentation": 7.5, "finalStageCompleteness": 7.7, "multiviewConsistency": 8 }, "plugin": { "taskFidelity": 8.2, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.1, "materialsAndReadability": 8, "surfaceFinishAndShading": 8.1, "materialTextureQuality": 7.7, "lightingAndPresentation": 7.8, "finalStageCompleteness": 8.2, "multiviewConsistency": 8.2 } }, "result": { "winner": "B", "confidence": 0.78, "scores": { "A": { "taskFidelity": 8.1, "silhouetteAndProportion": 7.3, "constructionPlausibility": 7.8, "craftsmanshipAndDetail": 7.6, "materialsAndReadability": 7.4, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 7.1, "lightingAndPresentation": 7.5, "finalStageCompleteness": 7.7, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8.2, "silhouetteAndProportion": 8.3, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8.1, "materialsAndReadability": 8, "surfaceFinishAndShading": 8.1, "materialTextureQuality": 7.7, "lightingAndPresentation": 7.8, "finalStageCompleteness": 8.2, "multiviewConsistency": 8.2 } }, "criterionResults": [ { "criterionId": "room_zones", "A": "pass", "B": "pass", "evidenceA": "Reading chair, footrest, rug and side table accompany a furnished desk with chair, lighting and stationery.", "evidenceB": "A substantial upholstered chair, ottoman, side table and lamp form a complete nook; the desk includes seating, task lighting and writing accessories." }, { "criterionId": "room_composition", "A": "pass", "B": "pass", "evidenceA": "Wall-based furniture leaves a clear walking route, although the small nook and extensive empty floor weaken focal hierarchy.", "evidenceB": "The window, rug, chair and bookcase establish a strong reading focal point, with the desk and decorated reverse wall balancing the open circulation area." }, { "criterionId": "room_secondary_finish", "A": "pass", "B": "pass", "evidenceA": "Continuous wall trim, framed art, populated shelving and a detailed low cabinet finish the secondary zones.", "evidenceB": "Populated shelving, framed pictures, cabinet fronts and desk hardware provide consistent secondary detail, though architectural trim is comparatively understated." }, { "criterionId": "room_support", "A": "pass", "B": "pass", "evidenceA": "Visible furniture legs reach the floor; shelf brackets, lamp bases and tabletop objects have plausible support without obvious major gaps.", "evidenceB": "Chair, ottoman, desk and side-table legs meet the floor; books rest on shelves and the task lamp has a visible wall mounting." }, { "criterionId": "room_materials", "A": "pass", "B": "pass", "evidenceA": "Wood grain, pale fabric, brass lamps and plaster are distinguishable, but the highly reflective floor and similar wood finishes reduce naturalism.", "evidenceB": "Matte upholstery and curtains contrast clearly with wood furniture, metallic fixtures and plaster; the wood floor has restrained surface variation." }, { "criterionId": "room_lighting", "A": "pass", "B": "pass", "evidenceA": "Warm practicals and window illumination keep secondary furniture readable, although the bright floor reflection competes with the nook.", "evidenceB": "Window shadows establish depth and warm task lighting reveals the desktop; upper walls and shelving are somewhat dim but remain readable." } ], "majorDefects": { "A": [ "The reading nook is visually underscaled against the large empty floor area.", "Strong floor gloss and uniformly warm wood surfaces weaken material realism." ], "B": [ "Architectural trim is sparse relative to the requested finished interior.", "Dim upper walls and shelf recesses reduce secondary detail visibility." ] }, "rationale": "B wins through a stronger reading-nook silhouette, more convincing upholstery, restrained floor finish and better-balanced furnishing clusters. A offers stronger architectural trim and brighter secondary views, but its distant nook and conspicuous floor reflection weaken the overall composition." } } ] }, "resourcePolicy": { "concurrency": 4, "cpuHardCapPercent": null }, "sharedResourceGuidanceHash": null, "limitation": "One generation per condition, medium effort, 30-minute generation limit. Two counterbalanced Astra reviews. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 1, "candidate": 1, "tie": 0 }, "vanilla": { "score": 96, "rawScore": 96, "hardGate": false, "rawHardGate": false, "seconds": 982, "triangles": 225416, "images": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "565634d316c1e23856eb430cf1fcc9ccff031224e00cc95847736f086173a3d8", "reverse": "dceebccaafeaac997b9630374ce1ad92307def697a3eadfe2dfa2e9a3c41ef55", "detail": "5cdfe62e812a25fe1f9afc9a66f31013fa4ead998fd3ccc00e90f71d94efb12b" }, "imageDimensions": { "hero": [ 1536, 864 ], "reverse": [ 1536, 864 ], "detail": [ 1536, 864 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "1782c1b39d5c5ad131d3b48c9cefac12df425434a862f252a44e1fe8245e60cc", "reverse": "5acd40e4f1fa1c33b64e5ab98afc18ee11d65a0ee9cc72c0a509368c6717b43f", "detail": "70cb5e9fe724a238bb2b8a61c21facd1e9deefb2fd976a94bee651689d2690b0" }, "originalImageDimensions": { "hero": [ 512, 288 ], "reverse": [ 512, 288 ], "detail": [ 512, 288 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--claude-opus-5-5--vanilla\\decorated_reading_room-r01\\asset.blend", "sourceSha256": "8bbc3123efff60000b205926c534ba07b02adca242161bd3273d956c69219085", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "ReadingRoom", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 17, "type": "PERSP", "location": [ 2.299999952316284, -2.200000047683716, 1.4500000476837158 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneReverse", "lens": 19, "type": "PERSP", "location": [ -1.25, 2.200000047683716, 1.5 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneDetail", "lens": 30, "type": "PERSP", "location": [ 1.6200000047683716, -0.550000011920929, 1.3799999952316284 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 } ], "lights": [ { "name": "Sun_LateAfternoon", "type": "SUN", "energy": 6, "color": [ 1, 0.7200000286102295, 0.44999998807907104 ], "hidden": false }, { "name": "WindowSkyFill", "type": "AREA", "energy": 120, "color": [ 0.800000011920929, 0.8600000143051147, 1 ], "hidden": false }, { "name": "TableLamp_Light", "type": "POINT", "energy": 35, "color": [ 1, 0.699999988079071, 0.4000000059604645 ], "hidden": false }, { "name": "DeskLamp_Light", "type": "SPOT", "energy": 30, "color": [ 1, 0.7400000095367432, 0.44999998807907104 ], "hidden": false }, { "name": "Pendant_Light", "type": "POINT", "energy": 25, "color": [ 1, 0.7200000286102295, 0.41999998688697815 ], "hidden": false }, { "name": "WallSconce_Light", "type": "POINT", "energy": 15, "color": [ 1, 0.699999988079071, 0.4000000059604645 ], "hidden": false }, { "name": "Ceiling_BounceFill", "type": "AREA", "energy": 60, "color": [ 1, 0.8799999952316284, 0.75 ], "hidden": false } ], "volumeMaterials": [], "world": "World_LateAfternoonSky", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": 0, "gamma": 1 }, "resolution": [ 1920, 1080, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-opus-5-5--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 8.052, "sha256": "565634d316c1e23856eb430cf1fcc9ccff031224e00cc95847736f086173a3d8" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-opus-5-5--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 7.322, "sha256": "dceebccaafeaac997b9630374ce1ad92307def697a3eadfe2dfa2e9a3c41ef55" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-opus-5-5--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 7.108, "sha256": "5cdfe62e812a25fe1f9afc9a66f31013fa4ead998fd3ccc00e90f71d94efb12b" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 864 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "1128 invalid vertices, degenerate faces, or zero-length edges" } ], "rawFailedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "1128 invalid vertices, degenerate faces, or zero-length edges" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--claude-opus-5-5--vanilla\\decorated_reading_room-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "e652c4618a296cb4e0811670e827077a022b47165b7c3258642661d38a8405e6", "asset.blend": "8bbc3123efff60000b205926c534ba07b02adca242161bd3273d956c69219085" }, "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2206278b2ba4509fe6848dc73cb77a769b4d4c07cb6845c90974f8af8705768f", "agentVersion": "2.1.288 (Claude Code)", "codexTransport": null, "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 7 }, "plugin": { "score": 96, "rawScore": 94, "hardGate": false, "rawHardGate": false, "seconds": 953, "triangles": 337768, "images": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "bd82e6580173af151653c303d184c2bd94dce2e776b3fee03ad13a024447cdb3", "reverse": "104c96f08ee543bb574a94f0df9f1431a64b5566375aaa623f8a27eee30d24e9", "detail": "768ba90b041ef376bc41b4c72eb42b008d339db23711f9cba6d639f336191052" }, "imageDimensions": { "hero": [ 1536, 960 ], "reverse": [ 1536, 960 ], "detail": [ 1536, 960 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/plugin/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/plugin/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-opus-5-5--r01/plugin/original/detail.png" }, "originalImageHashes": { "hero": "6a528194408ec2fbb711bd63f7bcb74884f02eccff85032be2811b6acccc5bec", "reverse": "6b1cc9d7901444bd4e034f2026a3627c5092fb76ab4d20b6dd4e9141ac79be25", "detail": "e85ff1c768cf4114df92b5009dd83ed24cbd4c5db25990600bb79b7f80fff4fe" }, "originalImageDimensions": { "hero": [ 512, 320 ], "reverse": [ 512, 320 ], "detail": [ 512, 320 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--claude-opus-5-5--plugin\\decorated_reading_room-r01\\asset.blend", "sourceSha256": "50174331ce9c316badba4ac7901293a25bfe449aa4021c715df532728714cb91", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "ReadingRoom", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 20, "type": "PERSP", "location": [ 5.550000190734863, 0.41999998688697815, 1.4800000190734863 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneReverse", "lens": 18, "type": "PERSP", "location": [ 5.099999904632568, 4.739999771118164, 1.850000023841858 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneDetail", "lens": 30, "type": "PERSP", "location": [ 1.850000023841858, 2.1500000953674316, 1.3799999952316284 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 } ], "lights": [ { "name": "Practical_TableLamp", "type": "POINT", "energy": 55, "color": [ 1, 0.699999988079071, 0.41999998688697815 ], "hidden": false }, { "name": "Practical_DeskSconce", "type": "SPOT", "energy": 30, "color": [ 1, 0.7200000286102295, 0.44999998807907104 ], "hidden": false }, { "name": "Practical_DeskSconceGlow", "type": "POINT", "energy": 6, "color": [ 1, 0.7200000286102295, 0.44999998807907104 ], "hidden": false }, { "name": "Sun_LateAfternoon", "type": "SUN", "energy": 9, "color": [ 1, 0.7799999713897705, 0.550000011920929 ], "hidden": false }, { "name": "Sky_WindowBack", "type": "AREA", "energy": 110, "color": [ 0.7799999713897705, 0.8600000143051147, 1 ], "hidden": false }, { "name": "Sky_WindowRight", "type": "AREA", "energy": 110, "color": [ 0.7799999713897705, 0.8600000143051147, 1 ], "hidden": false }, { "name": "Fill_CeilingBounce", "type": "AREA", "energy": 20, "color": [ 1, 0.8799999952316284, 0.7400000095367432 ], "hidden": false } ], "volumeMaterials": [], "world": "World_LateAfternoonSky", "colorManagement": { "viewTransform": "AgX", "look": "None", "exposure": 0, "gamma": 1 }, "resolution": [ 1920, 1200, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-opus-5-5--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 8.319, "sha256": "bd82e6580173af151653c303d184c2bd94dce2e776b3fee03ad13a024447cdb3" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-opus-5-5--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 8.281, "sha256": "104c96f08ee543bb574a94f0df9f1431a64b5566375aaa623f8a27eee30d24e9" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-opus-5-5--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 8.138, "sha256": "768ba90b041ef376bc41b4c72eb42b008d339db23711f9cba6d639f336191052" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 960 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": false, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "2832 invalid vertices, degenerate faces, or zero-length edges" } ], "rawFailedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "2832 invalid vertices, degenerate faces, or zero-length edges" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "40.000m maximum extent; limit 18m" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--claude-opus-5-5--plugin\\decorated_reading_room-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "bc732fc37308f0166c515d4194c2680cb900640bb03a19ce2e97c22b35185adf", "asset.blend": "50174331ce9c316badba4ac7901293a25bfe449aa4021c715df532728714cb91" }, "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2206278b2ba4509fe6848dc73cb77a769b4d4c07cb6845c90974f8af8705768f", "agentVersion": "2.1.288 (Claude Code)", "codexTransport": null, "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 7 }, "criteria": [ { "id": "room_zones", "label": "room zones", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_composition", "label": "room composition", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_secondary_finish", "label": "room secondary finish", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_support", "label": "room support", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_materials", "label": "room materials", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_lighting", "label": "room lighting", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "Review 1: the no-plugin result wins through more developed architectural trim, stronger material separation and consistently finished secondary furniture. the plugin result offers a convincing, intimate reading nook, but its flatter wall decoration and subdued shading make the overall finish less refined. the no-plugin result's oversized empty center and conspicuous floor glare slightly weaken its presentation. Review 2: the plugin result wins through a stronger reading-nook silhouette, more convincing upholstery, restrained floor finish and better-balanced furnishing clusters. the no-plugin result offers stronger architectural trim and brighter secondary views, but its distant nook and conspicuous floor reflection weaken the overall composition." }, { "id": "decorated_reading_room--claude-sonnet-5-5--r01", "task": "decorated_reading_room", "taskTitle": "Complete decorated reading room", "model": "claude-sonnet-5-5", "modelTitle": "Sonnet 5.5", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 2, "rawReview": { "taskId": "decorated_reading_room", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "plugin", "plugin" ], "criterionMajorities": [ { "criterionId": "room_zones", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_composition", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_secondary_finish", "baselinePasses": 1, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_materials", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_lighting", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "room_zones", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_composition", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_secondary_finish", "baselinePasses": 1, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "room_materials", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "plugin", "scoresByMode": { "vanilla": { "taskFidelity": 7, "silhouetteAndProportion": 6.5, "constructionPlausibility": 7, "craftsmanshipAndDetail": 5.5, "materialsAndReadability": 5.5, "surfaceFinishAndShading": 6.5, "materialTextureQuality": 4.5, "lightingAndPresentation": 7, "finalStageCompleteness": 6, "multiviewConsistency": 7 }, "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.5, "lightingAndPresentation": 8, "finalStageCompleteness": 8.5, "multiviewConsistency": 8 } }, "result": { "winner": "B", "confidence": 0.94, "scores": { "A": { "taskFidelity": 7, "silhouetteAndProportion": 6.5, "constructionPlausibility": 7, "craftsmanshipAndDetail": 5.5, "materialsAndReadability": 5.5, "surfaceFinishAndShading": 6.5, "materialTextureQuality": 4.5, "lightingAndPresentation": 7, "finalStageCompleteness": 6, "multiviewConsistency": 7 }, "B": { "taskFidelity": 9, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.5, "lightingAndPresentation": 8, "finalStageCompleteness": 8.5, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "room_zones", "A": "pass", "B": "pass", "evidenceA": "Armchair, rug and side-table lamp establish the nook; a separate furnished desk has a chair, lighting and writing materials.", "evidenceB": "Upholstered chair, ottoman, throw and side table complete the nook; the desk includes storage, seating, books, stationery and lighting." }, { "criterionId": "room_composition", "A": "pass", "B": "pass", "evidenceA": "Window-centered seating and the opposite desk leave a clear central route, although decoration clusters are sparse.", "evidenceB": "Rug, artwork, curtains and plant frame the nook; stocked shelving anchors the desk zone while the center remains open." }, { "criterionId": "room_secondary_finish", "A": "pass", "B": "pass", "evidenceA": "Reverse view shows a furnished bookcase, decorated desk shelves, framed artwork and architectural trim, albeit with simplified detailing.", "evidenceB": "Bookcases contain varied arrangements, desk drawers have hardware, and wall shelving and framed decorations extend the finish into secondary zones." }, { "criterionId": "room_support", "A": "pass", "B": "pass", "evidenceA": "Visible furniture legs reach the floor; lamps, books and pots sit on supporting surfaces, and wall shelves show brackets.", "evidenceB": "Desk legs and drawer pedestal provide clear support; lamps, books and stationery rest on surfaces, with no major visible floating parts." }, { "criterionId": "room_materials", "A": "fail", "B": "pass", "evidenceA": "Extremely dense parallel striping across the floor and desk overwhelms plausible oak grain scale; fabric and plaster have limited surface distinction.", "evidenceB": "Broader floorboards, wood grain, folded curtains, soft upholstery and matte walls read distinctly; restrained warm metal accents complement them." }, { "criterionId": "room_lighting", "A": "pass", "B": "pass", "evidenceA": "Warm window illumination and practical lamps keep both zones readable, with visible floor shadows; the desktop close-up is somewhat dim.", "evidenceB": "Window light and practical pools illuminate both zones while preserving wood detail, fabric folds and furniture contact shadows." } ], "majorDefects": { "A": [ "Overly dense, repetitive wood striping weakens material realism across large surfaces.", "Sparse accessories and simplified upholstery, shelving contents and furniture detailing reduce the sense of a polished finished room." ], "B": [ "The large foreground plant partially obscures the nook in the hero view.", "Some desk and chair components retain fairly plain, squared profiles." ] }, "rationale": "B delivers a more complete and convincing interior through richer nook furnishings, better-stocked shelving, stronger decoration clusters and more believable wood scale. A establishes both functional zones and plausible supports, but its repetitive textures and simplified detailing substantially weaken the polished finish." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "plugin", "scoresByMode": { "plugin": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 8.5 }, "vanilla": { "taskFidelity": 7.5, "silhouetteAndProportion": 7, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 6.5, "materialsAndReadability": 6, "surfaceFinishAndShading": 6.5, "materialTextureQuality": 5, "lightingAndPresentation": 8, "finalStageCompleteness": 6.5, "multiviewConsistency": 7.5 } }, "result": { "winner": "A", "confidence": 0.93, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.5, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 8.5 }, "B": { "taskFidelity": 7.5, "silhouetteAndProportion": 7, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 6.5, "materialsAndReadability": 6, "surfaceFinishAndShading": 6.5, "materialTextureQuality": 5, "lightingAndPresentation": 8, "finalStageCompleteness": 6.5, "multiviewConsistency": 7.5 } }, "criterionResults": [ { "criterionId": "room_zones", "A": "pass", "B": "pass", "evidenceA": "Upholstered chair, ottoman, rug and reading lamp complement a furnished desk with storage and chair.", "evidenceB": "Chair, rug and side lamp establish a usable nook; desk includes a chair, task lamp and writing materials." }, { "criterionId": "room_composition", "A": "pass", "B": "pass", "evidenceA": "Rug, curtains, artwork and plant frame the nook; bookshelf and wall shelf anchor the desk, with open circulation between zones.", "evidenceB": "Window and rug identify the nook, while shelving groups the desk area; the central walking route remains clear." }, { "criterionId": "room_secondary_finish", "A": "pass", "B": "fail", "evidenceA": "Populated bookcase, drawer fronts, layered desktop objects, framed art and trim carry detail into secondary views.", "evidenceB": "Sparse bookcase compartments, a largely empty long wall ledge and simplified secondary furniture leave substantial areas feeling underdeveloped." }, { "criterionId": "room_support", "A": "pass", "B": "pass", "evidenceA": "Visible desk legs, drawer pedestal, shelf brackets and tabletop contacts provide plausible support; no major floating objects are evident.", "evidenceB": "Desk legs, chair frame, shelf brackets and lamp bases visibly support their loads without clear major gaps." }, { "criterionId": "room_materials", "A": "pass", "B": "fail", "evidenceA": "Wood boards and furniture, soft upholstery and curtains, matte walls and metallic hardware have distinct, restrained responses.", "evidenceB": "Dense parallel striping overwhelms the floor and desk, reading as repeated grooves rather than convincing oak grain; fabric surfaces remain comparatively generic." }, { "criterionId": "room_lighting", "A": "pass", "B": "pass", "evidenceA": "Warm ambient and practical illumination retain wood detail, soft contact shadows and readable shelving.", "evidenceB": "Window light produces readable shadows across the nook and floor while the desk and reverse wall remain legible." } ], "majorDefects": { "A": [ "Oversized plant crowds the nook and partially obscures it in the reverse view.", "Large desk-side wall areas receive relatively little decoration." ], "B": [ "Excessively repetitive wood striping weakens material realism across large surfaces.", "Sparse shelving and simplified furniture reduce the sense of a fully finished interior.", "Angular lamp shade and stiff curtain folds weaken the polished smooth finish." ] }, "rationale": "A delivers a more complete, layered room with richer shelving, better upholstery, refined furniture details and more convincing wood. B has clear circulation and effective warm lighting, but repetitive textures and underdeveloped secondary furnishings noticeably reduce its finish quality." } } ] }, "resourcePolicy": { "concurrency": 4, "cpuHardCapPercent": null }, "sharedResourceGuidanceHash": null, "limitation": "One generation per condition, medium effort, 30-minute generation limit. Two counterbalanced Astra reviews. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 0, "candidate": 2, "tie": 0 }, "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 863, "triangles": 159460, "images": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "f76514e55a6da6a68db5b57a696a0ff94f38e2e4009c9391914b5128a9a0b0d8", "reverse": "72efcb409d8875e3ccfb7d62583980f966c58e692ea4f898016c47baf6626e40", "detail": "3fe302f045ea17741365c9067bed31076ba5a6b00404a92a3dd8840f80e780e6" }, "imageDimensions": { "hero": [ 1536, 864 ], "reverse": [ 1536, 864 ], "detail": [ 1536, 864 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "a56877790d4cfa7e76b19ef08c1ae4f1062d31e4bbd57df9c267205b1575b4ae", "reverse": "e37263a3310e9eea7e697434b78a0f35bc39ca944d3cb4950ab57d3448a010ca", "detail": "8022033823958716cd8e46088680b119fa33f5b507714fb340be4d83dad66d82" }, "originalImageDimensions": { "hero": [ 512, 288 ], "reverse": [ 512, 288 ], "detail": [ 512, 288 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--claude-sonnet-5-5--vanilla\\decorated_reading_room-r01\\asset.blend", "sourceSha256": "de19b617b0c0a07266d325dc002f50011b9066d23d1215fe6ba413fffd871358", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "ReadingRoom", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 22, "type": "PERSP", "location": [ 2.299999952316284, -1.899999976158142, 1.350000023841858 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneReverse", "lens": 20, "type": "PERSP", "location": [ -2.75, -2.25, 1.5 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneDetail", "lens": 30, "type": "PERSP", "location": [ 1.2000000476837158, -0.30000001192092896, 1.25 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 } ], "lights": [ { "name": "LateAfternoonSun", "type": "SUN", "energy": 7, "color": [ 1, 0.6800000071525574, 0.36000001430511475 ], "hidden": false }, { "name": "WindowSkyFill", "type": "AREA", "energy": 60, "color": [ 0.75, 0.8199999928474426, 1 ], "hidden": false }, { "name": "NookLampPractical", "type": "POINT", "energy": 60, "color": [ 1, 0.6200000047683716, 0.30000001192092896 ], "hidden": false }, { "name": "DeskLampPractical", "type": "POINT", "energy": 25, "color": [ 1, 0.699999988079071, 0.4000000059604645 ], "hidden": false }, { "name": "CeilingBounceFill", "type": "AREA", "energy": 30, "color": [ 1, 0.8199999928474426, 0.6200000047683716 ], "hidden": false }, { "name": "ReverseDoorFill", "type": "AREA", "energy": 25, "color": [ 1, 0.8500000238418579, 0.6499999761581421 ], "hidden": false }, { "name": "BookcaseWash", "type": "AREA", "energy": 35, "color": [ 1, 0.800000011920929, 0.550000011920929 ], "hidden": false } ], "volumeMaterials": [], "world": "WarmDusk", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": -0.30000001192092896, "gamma": 1 }, "resolution": [ 1920, 1080, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-sonnet-5-5--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 7.436, "sha256": "f76514e55a6da6a68db5b57a696a0ff94f38e2e4009c9391914b5128a9a0b0d8" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-sonnet-5-5--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 7.016, "sha256": "72efcb409d8875e3ccfb7d62583980f966c58e692ea4f898016c47baf6626e40" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-sonnet-5-5--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 7.131, "sha256": "3fe302f045ea17741365c9067bed31076ba5a6b00404a92a3dd8840f80e780e6" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 864 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--claude-sonnet-5-5--vanilla\\decorated_reading_room-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "7197104ae89d1700000cf8d985be4e980e564348e34232e32e479ead69d51363", "asset.blend": "de19b617b0c0a07266d325dc002f50011b9066d23d1215fe6ba413fffd871358" }, "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2206278b2ba4509fe6848dc73cb77a769b4d4c07cb6845c90974f8af8705768f", "agentVersion": "2.1.288 (Claude Code)", "codexTransport": null, "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 7 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 971, "triangles": 202308, "images": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "9ad0b93cb72a1aa266732d3016043f89c74dda46304775823faf3d98d53b62bd", "reverse": "8d8d0e4664e49f4a9449d35b4d68721538705d18c90f7606708037bb2bffdf1c", "detail": "5d8b09fd4ba42177b7a226d1f7626fae49cef662d154ac82f714a10d5777980b" }, "imageDimensions": { "hero": [ 1536, 864 ], "reverse": [ 1536, 864 ], "detail": [ 1536, 864 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/plugin/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/plugin/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/decorated_reading_room--claude-sonnet-5-5--r01/plugin/original/detail.png" }, "originalImageHashes": { "hero": "c8fbf633688f972fb5cdc4324007492de1ab3b5dfff57b4e0086859723f093be", "reverse": "0835104efa651f35a096e69d47f02ae78dc2f624660639cae94f0a7a7b4d6b89", "detail": "db1f04416698dc8141f0be61eb7e9a0ee09a8bbf1e3f648fd5d8f4b78f5e9081" }, "originalImageDimensions": { "hero": [ 512, 288 ], "reverse": [ 512, 288 ], "detail": [ 512, 288 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--claude-sonnet-5-5--plugin\\decorated_reading_room-r01\\asset.blend", "sourceSha256": "a52cd2b1a7c746644366963b2b6ddeb8955ddd3c08372177600990ca1401353d", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "ReadingRoom", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 22, "type": "PERSP", "location": [ -0.800000011920929, -2.299999952316284, 1.600000023841858 ], "clipStart": 0.05000000074505806, "clipEnd": 40 }, { "name": "SceneReverse", "lens": 24, "type": "PERSP", "location": [ 2.799999952316284, -2.4000000953674316, 1.7000000476837158 ], "clipStart": 0.05000000074505806, "clipEnd": 40 }, { "name": "SceneDetail", "lens": 32, "type": "PERSP", "location": [ -0.949999988079071, -1.350000023841858, 1.2999999523162842 ], "clipStart": 0.05000000074505806, "clipEnd": 40 } ], "lights": [ { "name": "LateAfternoonSun", "type": "SUN", "energy": 6, "color": [ 1, 0.7799999713897705, 0.550000011920929 ], "hidden": false }, { "name": "WindowSkyFill", "type": "AREA", "energy": 260, "color": [ 0.75, 0.8500000238418579, 1 ], "hidden": false }, { "name": "CeilingBounceFill", "type": "AREA", "energy": 90, "color": [ 1, 0.8799999952316284, 0.7200000286102295 ], "hidden": false }, { "name": "DeskLamp_PracticalLight", "type": "POINT", "energy": 20, "color": [ 1, 0.6800000071525574, 0.3799999952316284 ], "hidden": false }, { "name": "TableLamp_PracticalLight", "type": "POINT", "energy": 55, "color": [ 1, 0.6800000071525574, 0.3799999952316284 ], "hidden": false } ], "volumeMaterials": [], "world": "SkyWorld", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": 0.5, "gamma": 1 }, "resolution": [ 1280, 720, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-sonnet-5-5--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 7.585, "sha256": "9ad0b93cb72a1aa266732d3016043f89c74dda46304775823faf3d98d53b62bd" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-sonnet-5-5--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 7.381, "sha256": "8d8d0e4664e49f4a9449d35b4d68721538705d18c90f7606708037bb2bffdf1c" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\decorated_reading_room--claude-sonnet-5-5--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 7.189, "sha256": "5d8b09fd4ba42177b7a226d1f7626fae49cef662d154ac82f714a10d5777980b" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 864 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": false, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\decorated_reading_room--claude-sonnet-5-5--plugin\\decorated_reading_room-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "340c13121f49b24ded9e4cf22b29e9e792d8a21fb5c14f6027dd26fcfc780190", "asset.blend": "a52cd2b1a7c746644366963b2b6ddeb8955ddd3c08372177600990ca1401353d" }, "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2206278b2ba4509fe6848dc73cb77a769b4d4c07cb6845c90974f8af8705768f", "agentVersion": "2.1.288 (Claude Code)", "codexTransport": null, "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 7 }, "criteria": [ { "id": "room_zones", "label": "room zones", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_composition", "label": "room composition", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_secondary_finish", "label": "room secondary finish", "vanilla": { "pass": 1, "fail": 1, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_support", "label": "room support", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_materials", "label": "room materials", "vanilla": { "pass": 0, "fail": 2, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "room_lighting", "label": "room lighting", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "Review 1: the plugin result delivers a more complete and convincing interior through richer nook furnishings, better-stocked shelving, stronger decoration clusters and more believable wood scale. the no-plugin result establishes both functional zones and plausible supports, but its repetitive textures and simplified detailing substantially weaken the polished finish. Review 2: the plugin result delivers a more complete, layered room with richer shelving, better upholstery, refined furniture details and more convincing wood. the no-plugin result has clear circulation and effective warm lighting, but repetitive textures and underdeveloped secondary furnishings noticeably reduce its finish quality." }, { "id": "night_market_courtyard--gpt-6-astra--r01", "task": "night_market_courtyard", "taskTitle": "Complete low-poly night-market courtyard", "model": "gpt-6-astra", "modelTitle": "GPT 6 Astra", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 0, "rawReview": null, "resourcePolicy": { "concurrency": 4, "cpuHardCapPercent": null }, "sharedResourceGuidanceHash": null, "limitation": "One generation per condition, medium effort, 30-minute generation limit. Visual reviews unavailable; no preference inferred. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 0, "candidate": 0, "tie": 0 }, "vanilla": { "score": 96, "rawScore": 92.67, "hardGate": false, "rawHardGate": false, "seconds": 1172, "triangles": 102630, "images": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "15da22ec96147511c0c179c22e85437e8a1c6977df31a135e24e23ef300492f5", "reverse": "bbef058baf52aae66c9037f4eac2a226b3799ec029d784a57cc44a292c28f57c", "detail": "4320d1b5da2dc4a904484bf39613fe5b1f62a0ff321bde8b3fa775f337009dce" }, "imageDimensions": { "hero": [ 1536, 1536 ], "reverse": [ 1536, 1536 ], "detail": [ 1536, 1536 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "57e020c5ca9905c95f9b674fbf92a7f4ccc1a9c334d25d6e962c3f031f0dbb57", "reverse": "476200771162b2786df41ee58a9370d4bc70edb8cd355541786478ebd41c7234", "detail": "257dadab583a260a49ebfd702378133f4019db034ef51603a5e4d4d1044c765d" }, "originalImageDimensions": { "hero": [ 512, 512 ], "reverse": [ 512, 512 ], "detail": [ 512, 512 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--gpt-6-astra--vanilla\\night_market_courtyard-r01\\asset.blend", "sourceSha256": "04731fad72c142443767573ba2f28cecea578746c0e362cc6aa6de94bfe50c65", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 45, "type": "ORTHO", "location": [ 13, -18, 14.5 ], "clipStart": 0.10000000149011612, "clipEnd": 1000 }, { "name": "SceneReverse", "lens": 45, "type": "ORTHO", "location": [ -12, 17, 15 ], "clipStart": 0.10000000149011612, "clipEnd": 1000 }, { "name": "SceneDetail", "lens": 45, "type": "ORTHO", "location": [ -9, -11, 8 ], "clipStart": 0.10000000149011612, "clipEnd": 1000 } ], "lights": [ { "name": "Lantern warm pool", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.001", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.002", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.003", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.004", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.005", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.006", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.007", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.008", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.009", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.010", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.011", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.012", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.013", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.014", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.015", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.016", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Lantern warm pool.017", "type": "POINT", "energy": 45, "color": [ 1, 0.4300000071525574, 0.11999999731779099 ], "hidden": false }, { "name": "Stall warm counter light", "type": "POINT", "energy": 65, "color": [ 1, 0.6399999856948853, 0.30000001192092896 ], "hidden": false }, { "name": "Stall warm counter light.001", "type": "POINT", "energy": 65, "color": [ 1, 0.6399999856948853, 0.30000001192092896 ], "hidden": false }, { "name": "Stall warm counter light.002", "type": "POINT", "energy": 65, "color": [ 1, 0.6399999856948853, 0.30000001192092896 ], "hidden": false }, { "name": "Moon • broad cool key", "type": "AREA", "energy": 850, "color": [ 0.3199999928474426, 0.5199999809265137, 1 ], "hidden": false }, { "name": "Warm courtyard bounce", "type": "AREA", "energy": 260, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Reverse view moon fill", "type": "AREA", "energy": 650, "color": [ 0.3799999952316284, 0.5600000023841858, 1 ], "hidden": false } ], "volumeMaterials": [], "world": "Night air • blue ambient", "colorManagement": { "viewTransform": "AgX", "look": "None", "exposure": 0, "gamma": 1 }, "resolution": [ 1500, 1500, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6-astra--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 4.615, "sha256": "15da22ec96147511c0c179c22e85437e8a1c6977df31a135e24e23ef300492f5" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6-astra--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 3.968, "sha256": "bbef058baf52aae66c9037f4eac2a226b3799ec029d784a57cc44a292c28f57c" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6-astra--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 5.889, "sha256": "4320d1b5da2dc4a904484bf39613fe5b1f62a0ff321bde8b3fa775f337009dce" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1536 ], "samples": 48, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "568 invalid vertices, degenerate faces, or zero-length edges" } ], "rawFailedChecks": [ { "id": "semantic_part_coverage", "passed": false, "earned": 16.666666666666668, "possible": 20, "detail": "5/6 required semantic part groups found" }, { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "568 invalid vertices, degenerate faces, or zero-length edges" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http-evidence-repair\\night_market_courtyard--gpt-6-astra--vanilla\\evaluator-evidence", "evaluatorEvidenceRepaired": true, "artifactHashes": { "create_asset.py": "bbd64cfbe1c94d000b3c5bdb60d01b9c78b176de2bce67e69377633d98b991db", "asset.blend": "04731fad72c142443767573ba2f28cecea578746c0e362cc6aa6de94bfe50c65", "asset.glb": "8643f34f0f75d03aba0f7d6e36825c4f335c69a487869b836ef5801f99f36dec" }, "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2daae6f279a726c0e7cad74a138b8c87c9da2123f5c4dab94e082a1f423c1ea4", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6-astra", "modelsObserved": null, "rawScorerVersion": 7 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": false, "seconds": 1428, "triangles": 70860, "images": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "b69011fb86fd846517e8b6b97f5394f2e84aaf2f9103d157c978ac70a6630be4", "reverse": "a31af3076069df4d5b0838faeb72b9cb7534680d5fdb7f98729b84926d238bc2", "detail": "46ec22cdbcf9bf144054e336eaddc4d48d8da346030898bd183e3b88cb34dee8" }, "imageDimensions": { "hero": [ 1536, 1152 ], "reverse": [ 1536, 1152 ], "detail": [ 1536, 1152 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/plugin/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/plugin/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6-astra--r01/plugin/original/detail.png" }, "originalImageHashes": { "hero": "2bdd10c20abd3edaa8bcf1f96edeb9c0fcceeedc562ad8458f07e19fa762549f", "reverse": "97419e0ef4d43e2dd72a6d483b514b737d9268e640f30a06504fc6fcd054db28", "detail": "eb47db381777ad13ce1f9e4787ab48d1427e78ba15b604036407bbdd6c2582dc" }, "originalImageDimensions": { "hero": [ 512, 384 ], "reverse": [ 512, 384 ], "detail": [ 512, 384 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--gpt-6-astra--plugin\\night_market_courtyard-r01\\asset.blend", "sourceSha256": "5a2c463fe94fea5b3c6ed487dad0bae90ce537118bcfbf82760dd81b359d444f", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Moonlane • Night Market", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 50, "type": "ORTHO", "location": [ 13, -17, 13 ], "clipStart": 0.10000000149011612, "clipEnd": 200 }, { "name": "SceneReverse", "lens": 20, "type": "PERSP", "location": [ 0, 4.25, 4.800000190734863 ], "clipStart": 0.10000000149011612, "clipEnd": 200 }, { "name": "SceneDetail", "lens": 50, "type": "ORTHO", "location": [ 7.650000095367432, -13.899999618530273, 9.100000381469727 ], "clipStart": 0.10000000149011612, "clipEnd": 200 } ], "lights": [ { "name": "String 0 lantern 1 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 0 lantern 2 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 0 lantern 3 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 0 lantern 4 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 0 lantern 5 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 0 lantern 6 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 0 lantern 7 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 1 lantern 1 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 1 lantern 2 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 1 lantern 3 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 1 lantern 4 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 1 lantern 5 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 1 lantern 6 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 1 lantern 7 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 2 lantern 1 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 2 lantern 2 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 2 lantern 3 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 2 lantern 4 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 2 lantern 5 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 2 lantern 6 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "String 2 lantern 7 practical", "type": "POINT", "energy": 35, "color": [ 1, 0.47999998927116394, 0.1599999964237213 ], "hidden": false }, { "name": "Cool moon softbox", "type": "AREA", "energy": 900, "color": [ 0.36000001430511475, 0.5299999713897705, 1 ], "hidden": false }, { "name": "Warm southwest bounce", "type": "AREA", "energy": 500, "color": [ 1, 0.6399999856948853, 0.3400000035762787 ], "hidden": false }, { "name": "Cool reverse fill", "type": "AREA", "energy": 650, "color": [ 0.4300000071525574, 0.6600000262260437, 1 ], "hidden": false }, { "name": "Kitchen service lamp", "type": "AREA", "energy": 95, "color": [ 1, 0.47999998927116394, 0.18000000715255737 ], "hidden": false }, { "name": "Pottery service lamp", "type": "AREA", "energy": 100, "color": [ 1, 0.6499999761581421, 0.36000001430511475 ], "hidden": false }, { "name": "Florist service lamp", "type": "AREA", "energy": 110, "color": [ 1, 0.6700000166893005, 0.4399999976158142 ], "hidden": false } ], "volumeMaterials": [], "world": "Cool nocturnal sky", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": -0.20000000298023224, "gamma": 1 }, "resolution": [ 1600, 1200, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6-astra--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 2.798, "sha256": "b69011fb86fd846517e8b6b97f5394f2e84aaf2f9103d157c978ac70a6630be4" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6-astra--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 3.025, "sha256": "a31af3076069df4d5b0838faeb72b9cb7534680d5fdb7f98729b84926d238bc2" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6-astra--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 3.252, "sha256": "46ec22cdbcf9bf144054e336eaddc4d48d8da346030898bd183e3b88cb34dee8" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1152 ], "samples": 32, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [ { "id": "authored_camera_evidence", "passed": false, "earned": 0, "possible": 0, "detail": "All contracted scene cameras must render with available dependencies; this does not certify composition or finish" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http-evidence-repair\\night_market_courtyard--gpt-6-astra--plugin\\evaluator-evidence", "evaluatorEvidenceRepaired": true, "artifactHashes": { "create_asset.py": "5ff8099c94992ed4836fba0ea0cdb5893cd5db97631bc1a376476f06f0203325", "asset.blend": "5a2c463fe94fea5b3c6ed487dad0bae90ce537118bcfbf82760dd81b359d444f", "asset.glb": "12c32aae434f30be4f6e778102e58391a55c5e81cf4298fdcf935dc023df2346" }, "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2daae6f279a726c0e7cad74a138b8c87c9da2123f5c4dab94e082a1f423c1ea4", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6-astra", "modelsObserved": null, "rawScorerVersion": 7 }, "criteria": [], "note": "Complete paired visual evidence or valid blinded reviews were unavailable. This attempt remains in the matrix; no winner is assigned." }, { "id": "night_market_courtyard--gpt-6.1-sol--r01", "task": "night_market_courtyard", "taskTitle": "Complete low-poly night-market courtyard", "model": "gpt-6.1-sol", "modelTitle": "GPT 6.1 Sol", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 0, "rawReview": null, "resourcePolicy": { "concurrency": 4, "cpuHardCapPercent": null }, "sharedResourceGuidanceHash": null, "limitation": "One generation per condition, medium effort, 30-minute generation limit. Visual reviews unavailable; no preference inferred. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 0, "candidate": 0, "tie": 0 }, "vanilla": { "score": 96, "rawScore": 96, "hardGate": false, "rawHardGate": false, "seconds": 1259, "triangles": 91510, "images": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "ff86b279e543399f5dda8019453ef054740bef604ab531f4c804338ef85af119", "reverse": "72224248a377105f46c4b87279093aa53f57f732bbae059e82d25bd12e0f1e25", "detail": "62535f70a7b0c92c6d2eca4156b6457492fdcc2573409df786d787b4bfb0eb74" }, "imageDimensions": { "hero": [ 1536, 1344 ], "reverse": [ 1536, 1344 ], "detail": [ 1536, 1344 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "efc2ee2fbf2727c59e436676a7d4968357544d939c601cd4d12b7e39dd2a5de6", "reverse": "e933cf615b8975996c2c2b3fbed00ea1b55e2aeb5bd9b89d8aa2df40a6c7a039", "detail": "c6aeb9a8ad5ff0f09718e99e6241b350fd52911f0c27aae963fb6a6a055c9bd0" }, "originalImageDimensions": { "hero": [ 512, 448 ], "reverse": [ 512, 448 ], "detail": [ 512, 448 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--gpt-6.1-sol--vanilla\\night_market_courtyard-r01\\asset.blend", "sourceSha256": "8f1eaf1558e655a477fdf991581ac36da0bf07dea68c4c2bf8facd1ae9f923d3", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 45, "type": "ORTHO", "location": [ 13, -17, 14 ], "clipStart": 0.10000000149011612, "clipEnd": 1000 }, { "name": "SceneReverse", "lens": 16, "type": "PERSP", "location": [ 0.5, 4.179999828338623, 6.699999809265137 ], "clipStart": 0.10000000149011612, "clipEnd": 1000 }, { "name": "SceneDetail", "lens": 45, "type": "ORTHO", "location": [ -8.5, -9.5, 7.5 ], "clipStart": 0.10000000149011612, "clipEnd": 1000 } ], "lights": [ { "name": "Amber pool from string", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.001", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.002", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.003", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.004", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.005", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.006", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.007", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.008", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.009", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.010", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Amber pool from string.011", "type": "POINT", "energy": 60, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Warm facade sconce", "type": "POINT", "energy": 28, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Warm facade sconce.001", "type": "POINT", "energy": 28, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Warm facade sconce.002", "type": "POINT", "energy": 28, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Warm facade sconce.003", "type": "POINT", "energy": 28, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Table lantern pool", "type": "POINT", "energy": 12, "color": [ 1, 0.5199999809265137, 0.2199999988079071 ], "hidden": false }, { "name": "Stall inventory light", "type": "POINT", "energy": 55, "color": [ 1, 0.6800000071525574, 0.4000000059604645 ], "hidden": false }, { "name": "Stall inventory light.001", "type": "POINT", "energy": 55, "color": [ 1, 0.6800000071525574, 0.4000000059604645 ], "hidden": false }, { "name": "Stall inventory light.002", "type": "POINT", "energy": 55, "color": [ 1, 0.6800000071525574, 0.4000000059604645 ], "hidden": false }, { "name": "Cool moon area", "type": "AREA", "energy": 600, "color": [ 0.38999998569488525, 0.6200000047683716, 1 ], "hidden": false }, { "name": "Warm courtyard bounce", "type": "AREA", "energy": 260, "color": [ 1, 0.6600000262260437, 0.3700000047683716 ], "hidden": false }, { "name": "Cool reverse fill", "type": "AREA", "energy": 500, "color": [ 0.3499999940395355, 0.6600000262260437, 0.8500000238418579 ], "hidden": false }, { "name": "Gentle entry fill", "type": "AREA", "energy": 180, "color": [ 0.4399999976158142, 0.6899999976158142, 0.8500000238418579 ], "hidden": false } ], "volumeMaterials": [], "world": "World", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": 0, "gamma": 1 }, "resolution": [ 1600, 1400, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6.1-sol--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 4.104, "sha256": "ff86b279e543399f5dda8019453ef054740bef604ab531f4c804338ef85af119" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6.1-sol--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 4.742, "sha256": "72224248a377105f46c4b87279093aa53f57f732bbae059e82d25bd12e0f1e25" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6.1-sol--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 5.355, "sha256": "62535f70a7b0c92c6d2eca4156b6457492fdcc2573409df786d787b4bfb0eb74" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1344 ], "samples": 48, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "960 invalid vertices, degenerate faces, or zero-length edges" } ], "rawFailedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "960 invalid vertices, degenerate faces, or zero-length edges" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--gpt-6.1-sol--vanilla\\night_market_courtyard-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "2deab45078969bfd81013d40cf40b5c6381251c955f8d3134f6295a73a83407f", "asset.blend": "8f1eaf1558e655a477fdf991581ac36da0bf07dea68c4c2bf8facd1ae9f923d3", "asset.glb": "372d2cae8dae937d44c93c15303d38384e810df07bcb5b585bafb16a36a3e8c1" }, "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2daae6f279a726c0e7cad74a138b8c87c9da2123f5c4dab94e082a1f423c1ea4", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 7 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": false, "rawHardGate": false, "seconds": 1576, "triangles": 96880, "images": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--gpt-6.1-sol--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "60f72e4f14842e0322cae9cedccb5a243e4997d19f657900ca0a1502b576c779", "reverse": "ef95f6a12ec80a3c1701826b440adbe81304adbfa283211de749d73780780229", "detail": "7d2126b9084736f5df0c1a5e177281eb2d39641e32511cbc7641d5b584de9490" }, "imageDimensions": { "hero": [ 1536, 1152 ], "reverse": [ 1536, 1152 ], "detail": [ 1536, 1152 ] }, "originalImages": {}, "originalImageHashes": {}, "originalImageDimensions": {}, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--gpt-6.1-sol--plugin\\night_market_courtyard-r01\\asset.blend", "sourceSha256": "170cbdb1caeb142b9b401c11b08269c3953122a467bd54864ea14c9e732ee369", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Moonlight_Yard", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 50, "type": "ORTHO", "location": [ -14, -18, 15 ], "clipStart": 0.10000000149011612, "clipEnd": 150 }, { "name": "SceneReverse", "lens": 23, "type": "PERSP", "location": [ 9, 4.099999904632568, 9 ], "clipStart": 0.10000000149011612, "clipEnd": 150 }, { "name": "SceneDetail", "lens": 50, "type": "ORTHO", "location": [ 0, -7.699999809265137, 5.900000095367432 ], "clipStart": 0.10000000149011612, "clipEnd": 150 }, { "name": "SceneBackWork", "lens": 28, "type": "PERSP", "location": [ 1.7999999523162842, 4, 2.700000047683716 ], "clipStart": 0.10000000149011612, "clipEnd": 150 } ], "lights": [ { "name": "FrontFestoon_Lamp00_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "FrontFestoon_Lamp01_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "FrontFestoon_Lamp02_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "FrontFestoon_Lamp03_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "FrontFestoon_Lamp04_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "FrontFestoon_Lamp05_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "FrontFestoon_Lamp06_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "FrontFestoon_Lamp07_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "FrontFestoon_Lamp08_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp00_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp01_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp02_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp03_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp04_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp05_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp06_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp07_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "BackFestoon_Lamp08_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp00_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp01_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp02_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp03_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp04_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp05_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp06_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp07_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "DiagonalFestoon_Lamp08_WarmPool", "type": "POINT", "energy": 34, "color": [ 1, 0.5299999713897705, 0.20000000298023224 ], "hidden": false }, { "name": "Moon_Key", "type": "AREA", "energy": 550, "color": [ 0.3199999928474426, 0.46000000834465027, 1 ], "hidden": false }, { "name": "Courtyard_CoolFill", "type": "AREA", "energy": 340, "color": [ 0.3799999952316284, 0.550000011920929, 1 ], "hidden": false }, { "name": "Facade_WarmBounce", "type": "AREA", "energy": 170, "color": [ 1, 0.5699999928474426, 0.30000001192092896 ], "hidden": false }, { "name": "Food_TaskLight", "type": "AREA", "energy": 100, "color": [ 1, 0.6499999761581421, 0.33000001311302185 ], "hidden": false }, { "name": "Clay_TaskLight", "type": "AREA", "energy": 90, "color": [ 1, 0.7200000286102295, 0.41999998688697815 ], "hidden": false }, { "name": "Bloom_TaskLight", "type": "AREA", "energy": 110, "color": [ 1, 0.7099999785423279, 0.4000000059604645 ], "hidden": false }, { "name": "Seating_WarmPool", "type": "AREA", "energy": 180, "color": [ 1, 0.6499999761581421, 0.3499999940395355 ], "hidden": false }, { "name": "Reverse_UsefulFill", "type": "AREA", "energy": 300, "color": [ 0.3499999940395355, 0.5, 1 ], "hidden": false } ], "volumeMaterials": [], "world": "Indigo_Night", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": 0, "gamma": 1 }, "resolution": [ 1440, 1080, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6.1-sol--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 4.195, "sha256": "60f72e4f14842e0322cae9cedccb5a243e4997d19f657900ca0a1502b576c779" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6.1-sol--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 3.596, "sha256": "ef95f6a12ec80a3c1701826b440adbe81304adbfa283211de749d73780780229" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--gpt-6.1-sol--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 5.061, "sha256": "7d2126b9084736f5df0c1a5e177281eb2d39641e32511cbc7641d5b584de9490" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1152 ], "samples": 48, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": true, "failedChecks": [ { "id": "authored_camera_evidence", "passed": false, "earned": 0, "possible": 0, "detail": "All contracted scene cameras must render with available dependencies; this does not certify composition or finish" } ], "rawFailedChecks": [ { "id": "authored_camera_evidence", "passed": false, "earned": 0, "possible": 0, "detail": "All contracted scene cameras must render with available dependencies; this does not certify composition or finish" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--gpt-6.1-sol--plugin\\night_market_courtyard-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "65b69694b3c5ad88cde6ec43fa64c9100d3a5b2ae7521d88604b5f5efd4d07ac", "asset.blend": "170cbdb1caeb142b9b401c11b08269c3953122a467bd54864ea14c9e732ee369", "asset.glb": "95976eede38a5b60d81128946aaf9323edf14122019a39910787dac63adad659" }, "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2daae6f279a726c0e7cad74a138b8c87c9da2123f5c4dab94e082a1f423c1ea4", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 7 }, "criteria": [], "note": "Complete paired visual evidence or valid blinded reviews were unavailable. This attempt remains in the matrix; no winner is assigned." }, { "id": "night_market_courtyard--claude-opus-5-5--r01", "task": "night_market_courtyard", "taskTitle": "Complete low-poly night-market courtyard", "model": "claude-opus-5-5", "modelTitle": "Opus 5.5", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 2, "rawReview": { "taskId": "night_market_courtyard", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "vanilla", "vanilla" ], "criterionMajorities": [ { "criterionId": "market_stalls", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_construction", "baselinePasses": 1, "candidatePasses": 1, "judgeCount": 2 }, { "criterionId": "market_route", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_palette", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_light", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "market_stalls", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_construction", "baselinePasses": 1, "candidatePasses": 1, "judgeCount": 2 }, { "criterionId": "market_route", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_support", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_palette", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 7, "silhouetteAndProportion": 7.5, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 6.5, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 6.5, "lightingAndPresentation": 7.5, "finalStageCompleteness": 6.5, "multiviewConsistency": 8 }, "vanilla": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "result": { "winner": "B", "confidence": 0.86, "scores": { "A": { "taskFidelity": 7, "silhouetteAndProportion": 7.5, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 6.5, "materialsAndReadability": 7.5, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 6.5, "lightingAndPresentation": 7.5, "finalStageCompleteness": 6.5, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "market_stalls", "A": "pass", "B": "pass", "evidenceA": "Red food, teal pottery and purple flower stalls have visible counter inventory; flowers are clearest in the middle panel.", "evidenceB": "Food, ceramics and flower stalls have distinct canopies and dressed counters with additional shelving or containers." }, { "criterionId": "market_construction", "A": "pass", "B": "pass", "evidenceA": "Timber counters, canopy posts, shuttered facades and paved seating areas form a coherent low-poly courtyard.", "evidenceB": "Framed stalls, paneled counters, facade trim, entrance pillars and finished seating areas maintain intentional low-poly construction." }, { "criterionId": "market_route", "A": "pass", "B": "pass", "evidenceA": "An open central paved aisle connects the stalls while tables remain along the perimeter.", "evidenceB": "A clear entrance leads into a connected central courtyard, with seating and stalls positioned outside the main route." }, { "criterionId": "market_support", "A": "pass", "B": "pass", "evidenceA": "Visible goods rest on counters, crates sit on paving, tables have legs, and lantern cables connect to posts or walls.", "evidenceB": "Inventory sits on counters and shelves; benches, pedestal tables and lantern strings have visible supporting structures." }, { "criterionId": "market_palette", "A": "pass", "B": "pass", "evidenceA": "Colored canopies and contrasting food, pottery and floral shapes distinguish vendors within a warm timber and teal palette.", "evidenceB": "Vendor colors, inventory silhouettes and signage distinguish the stalls while repeating timber, plaster and teal accents." }, { "criterionId": "market_light", "A": "pass", "B": "pass", "evidenceA": "Warm lanterns and windows illuminate counters, paving and seating against a cool night background.", "evidenceB": "Warm lanterns and cool ambient light keep all three stalls, the entrance and reverse seating area legible." } ], "majorDefects": { "A": [ "Requested readable vendor signs are absent from the visible views.", "Seating props and stall detailing remain comparatively sparse and generic." ], "B": [] }, "rationale": "B more convincingly completes the market through readable market and vendor signage, richer stall dressing and stronger architectural finishing. A satisfies the core spatial and construction criteria, but its missing signage and simpler secondary details make it feel less finished. Neither shows a decisive support or route failure." } }, { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.3, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 }, "plugin": { "taskFidelity": 7.2, "silhouetteAndProportion": 8, "constructionPlausibility": 7.6, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.3, "lightingAndPresentation": 7.7, "finalStageCompleteness": 7.6, "multiviewConsistency": 8 } }, "result": { "winner": "A", "confidence": 0.76, "scores": { "A": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7.8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.3, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 }, "B": { "taskFidelity": 7.2, "silhouetteAndProportion": 8, "constructionPlausibility": 7.6, "craftsmanshipAndDetail": 7.5, "materialsAndReadability": 7.8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7.3, "lightingAndPresentation": 7.7, "finalStageCompleteness": 7.6, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "market_stalls", "A": "pass", "B": "pass", "evidenceA": "Food, pottery and flower stalls have distinct canopies, displayed inventory and visible vendor labels.", "evidenceB": "Three dressed stalls visibly display food, ceramic vessels and flowers beneath distinct colored canopies." }, { "criterionId": "market_construction", "A": "unclear", "B": "unclear", "evidenceA": "Visible counters, canopy frames and facade look finished, but the supplied views do not establish every stall's reverse construction.", "evidenceB": "Timber counters, facade and visible flower-stall reverse have intentional low-poly construction; other stall reverses remain insufficiently exposed." }, { "criterionId": "market_route", "A": "pass", "B": "pass", "evidenceA": "The entrance connects to a broad paved courtyard with seating and stalls positioned around its edges.", "evidenceB": "A continuous central aisle connects the entrance and vendors; peripheral seating leaves usable circulation." }, { "criterionId": "market_support", "A": "pass", "B": "pass", "evidenceA": "Visible merchandise rests on counters, furniture has grounded supports, and lanterns hang from overhead cords between structural elements.", "evidenceB": "Displays and crates rest on surfaces, benches and tables have visible legs, and lantern cords connect to poles or perimeter structures." }, { "criterionId": "market_palette", "A": "pass", "B": "pass", "evidenceA": "Red, teal and purple canopies, readable labels and differentiated inventory clearly separate vendors within a coherent plaster-and-timber palette.", "evidenceB": "Canopy colors and inventory distinguish vendors while teal shutters, warm timber and neutral paving unify the courtyard." }, { "criterionId": "market_light", "A": "pass", "B": "pass", "evidenceA": "Warm lanterns and windows contrast with the blue night surroundings; counters, paving and seating remain legible.", "evidenceB": "Warm illumination reveals all three vendors and seating against cool night surroundings, although canopy highlights are somewhat strong." } ], "majorDefects": { "A": [ "Not every stall reverse is exposed sufficiently to verify complete construction." ], "B": [ "No readable vendor or market signs are visible.", "Not every stall reverse is exposed sufficiently to verify complete construction." ] }, "rationale": "A wins through clearer vendor identification, readable signage and more varied visible prop dressing. B provides coherent construction and usable circulation, but missing visible signs weakens completion of the brief. Neither sheet fully proves all reverse surfaces." } } ] }, "resourcePolicy": { "concurrency": 4, "cpuHardCapPercent": null }, "sharedResourceGuidanceHash": null, "limitation": "One generation per condition, medium effort, 30-minute generation limit. Two counterbalanced Astra reviews. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 2, "candidate": 0, "tie": 0 }, "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 466, "triangles": 17787, "images": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "09e2ab6308c767e4f3f85626bef863e903b6d22ea2403d7e60cfeba938c832d8", "reverse": "6124c1ceadac52942e1ae5575c559db75119e507b39c1dd49bb16405f0b7e07e", "detail": "84468026beb4d4074ef281a23c02b17a55662b088022cb48df9b61acf0b2eaf5" }, "imageDimensions": { "hero": [ 1536, 864 ], "reverse": [ 1536, 864 ], "detail": [ 1536, 864 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "f30b31060360fbfceda1704244b2d692709da68d09510c07ac35b8292a21bade", "reverse": "f95f3c98b0767c80eb2329774ae3b14605d4e6396a8194625367303a8ef83695", "detail": "f62b33b53541a88fd48b5c64856c658ea7e55e7fdfdf98b9cf05378319ac7c5b" }, "originalImageDimensions": { "hero": [ 512, 288 ], "reverse": [ 512, 288 ], "detail": [ 512, 288 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--claude-opus-5-5--vanilla\\night_market_courtyard-r01\\asset.blend", "sourceSha256": "d96669eab36ad0a06e1534d4c08ed3198b9fb4a1c95164030aa2f31dd451ae02", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "NightMarketCourtyard", "engine": "BLENDER_EEVEE", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 26, "type": "PERSP", "location": [ -3.200000047683716, -12.5, 8.199999809265137 ], "clipStart": 0.10000000149011612, "clipEnd": 200 }, { "name": "SceneReverse", "lens": 22, "type": "PERSP", "location": [ 3, 4.300000190734863, 2.4000000953674316 ], "clipStart": 0.10000000149011612, "clipEnd": 200 }, { "name": "SceneDetail", "lens": 28, "type": "PERSP", "location": [ 1.2000000476837158, -0.699999988079071, 1.7999999523162842 ], "clipStart": 0.10000000149011612, "clipEnd": 200 } ], "lights": [ { "name": "Food_Bulb_A_Light", "type": "POINT", "energy": 70, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Food_Bulb_B_Light", "type": "POINT", "energy": 70, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Ceramics_Bulb_A_Light", "type": "POINT", "energy": 60, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Ceramics_Bulb_B_Light", "type": "POINT", "energy": 60, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Flowers_Bulb_A_Light", "type": "POINT", "energy": 70, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Seating_LampLight", "type": "POINT", "energy": 90, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_N_East_01_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_N_East_02_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_N_East_03_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_N_East_04_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_N_East_05_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_N_East_06_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SE_N_West_01_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SE_N_West_02_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SE_N_West_03_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SE_N_West_04_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SE_N_West_05_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SE_N_West_06_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_W_Mid_E_Mid_01_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_W_Mid_E_Mid_02_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_W_Mid_E_Mid_03_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_W_Mid_E_Mid_04_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_W_Mid_E_Mid_05_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_W_Mid_E_Mid_06_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_SE_01_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_SE_02_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_SE_03_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_SE_04_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_SE_05_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Lantern_SW_SE_06_Light", "type": "POINT", "energy": 45, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Moonlight", "type": "SUN", "energy": 0.3499999940395355, "color": [ 0.550000011920929, 0.6499999761581421, 1 ], "hidden": false }, { "name": "SkyFill_Cool", "type": "AREA", "energy": 120, "color": [ 0.44999998807907104, 0.550000011920929, 1 ], "hidden": false } ], "volumeMaterials": [], "world": "NightSky", "colorManagement": { "viewTransform": "AgX", "look": "None", "exposure": 0.30000001192092896, "gamma": 1 }, "resolution": [ 1600, 900, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-opus-5-5--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 1.81, "sha256": "09e2ab6308c767e4f3f85626bef863e903b6d22ea2403d7e60cfeba938c832d8" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-opus-5-5--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 1.134, "sha256": "6124c1ceadac52942e1ae5575c559db75119e507b39c1dd49bb16405f0b7e07e" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-opus-5-5--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 1.076, "sha256": "84468026beb4d4074ef281a23c02b17a55662b088022cb48df9b61acf0b2eaf5" } ], "device": { "requested": "OPTIX", "effective": "engine-managed", "devices": [] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "engine-managed", "denoiser": null, "quality": null, "prefilter": null, "gpu": null, "decisionReason": "Render engine is not Cycles.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 864 ], "samples": null, "timeLimitSeconds": null, "denoise": null, "compositing": false, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--claude-opus-5-5--vanilla\\night_market_courtyard-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "16f3df30e8b7236b765504103e172133793af4a1070693f533fa1ea839d52244", "asset.blend": "d96669eab36ad0a06e1534d4c08ed3198b9fb4a1c95164030aa2f31dd451ae02", "asset.glb": "4c349531b6e6a58c65672ea3690043d930a08bab518cc9f6b424ae586713ed59" }, "executionMode": "baseline", "guidanceHash": null, "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2daae6f279a726c0e7cad74a138b8c87c9da2123f5c4dab94e082a1f423c1ea4", "agentVersion": "2.1.288 (Claude Code)", "codexTransport": null, "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 7 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 375, "triangles": 17516, "images": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "251b059938919e523578a3b6d8f7aef23e86426d911b57fee1d0448dd67fbc33", "reverse": "d94e24cdb30675c034fef52ab58e76291a184f814e50c699cc9512b3c110651f", "detail": "db2d89b517464f7529e17798857434b9a6a177f5c43af7e84addd42f2df03303" }, "imageDimensions": { "hero": [ 1536, 864 ], "reverse": [ 1536, 864 ], "detail": [ 1536, 864 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/plugin/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/plugin/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-opus-5-5--r01/plugin/original/detail.png" }, "originalImageHashes": { "hero": "db5f83575130772e5f701a21048650bfc85cf6ebb9ad175163eb9fd546826166", "reverse": "8fff532991a6c59d049622f5bbf86211d21e6c0593d7e3949f7066f70ca21c31", "detail": "24a00d30af65b9dba37e6f3d76af65c7fb23391f2cd89224391a50e1cd39525e" }, "originalImageDimensions": { "hero": [ 512, 288 ], "reverse": [ 512, 288 ], "detail": [ 512, 288 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--claude-opus-5-5--plugin\\night_market_courtyard-r01\\asset.blend", "sourceSha256": "e3366ce28821bdc27af9262607d61c2d7a20707ff2d6279d4468a8ca8eb07d3e", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 24, "type": "PERSP", "location": [ 1.399999976158142, -10.800000190734863, 8.600000381469727 ], "clipStart": 0.10000000149011612, "clipEnd": 200 }, { "name": "SceneReverse", "lens": 20, "type": "PERSP", "location": [ 0.30000001192092896, 3.9000000953674316, 4.699999809265137 ], "clipStart": 0.10000000149011612, "clipEnd": 200 }, { "name": "SceneDetail", "lens": 24, "type": "PERSP", "location": [ -0.8999999761581421, -1.2000000476837158, 1.7999999523162842 ], "clipStart": 0.10000000149011612, "clipEnd": 200 } ], "lights": [ { "name": "Light_Moon_Cool", "type": "SUN", "energy": 0.25, "color": [ 0.550000011920929, 0.6800000071525574, 1 ], "hidden": false }, { "name": "Light_Lantern_S0_02", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S0_06", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S0_10", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S1_02", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S1_06", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S1_10", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S2_02", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S2_06", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S2_10", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S3_02", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S3_06", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S3_10", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S4_02", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S4_06", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Lantern_S4_10", "type": "POINT", "energy": 18, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "Light_Stall_Food", "type": "AREA", "energy": 90, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Light_Stall_Ceramics", "type": "AREA", "energy": 70, "color": [ 1, 0.75, 0.5 ], "hidden": false }, { "name": "Light_Stall_Flowers", "type": "AREA", "energy": 70, "color": [ 1, 0.699999988079071, 0.44999998807907104 ], "hidden": false }, { "name": "Light_Seating_Candle", "type": "AREA", "energy": 40, "color": [ 1, 0.550000011920929, 0.25 ], "hidden": false }, { "name": "Light_Sky_Fill_Cool", "type": "AREA", "energy": 70, "color": [ 0.44999998807907104, 0.6000000238418579, 1 ], "hidden": false } ], "volumeMaterials": [], "world": "World_NightSky", "colorManagement": { "viewTransform": "AgX", "look": "None", "exposure": 0.30000001192092896, "gamma": 1 }, "resolution": [ 1600, 900, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-opus-5-5--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 4.213, "sha256": "251b059938919e523578a3b6d8f7aef23e86426d911b57fee1d0448dd67fbc33" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-opus-5-5--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 3.909, "sha256": "d94e24cdb30675c034fef52ab58e76291a184f814e50c699cc9512b3c110651f" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-opus-5-5--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 3.931, "sha256": "db2d89b517464f7529e17798857434b9a6a177f5c43af7e84addd42f2df03303" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 864 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-http\\night_market_courtyard--claude-opus-5-5--plugin\\night_market_courtyard-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "a1ef1162fbbfed5bbf126f8c5f81dbc59037ae1bb93d0af1d79ce1aa4d34ed0f", "asset.blend": "e3366ce28821bdc27af9262607d61c2d7a20707ff2d6279d4468a8ca8eb07d3e", "asset.glb": "101088e0445ab91cba9cc3ed3aa1b20773923423f379a511e51ccd40d493bd47" }, "executionMode": "skills", "guidanceHash": null, "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2daae6f279a726c0e7cad74a138b8c87c9da2123f5c4dab94e082a1f423c1ea4", "agentVersion": "2.1.288 (Claude Code)", "codexTransport": null, "modelRequested": "claude-opus-5-5", "modelsObserved": [ "claude-opus-5-5" ], "rawScorerVersion": 7 }, "criteria": [ { "id": "market_stalls", "label": "market stalls", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "market_construction", "label": "market construction", "vanilla": { "pass": 1, "fail": 0, "unclear": 1 }, "plugin": { "pass": 1, "fail": 0, "unclear": 1 } }, { "id": "market_route", "label": "market route", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "market_support", "label": "market support", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "market_palette", "label": "market palette", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "market_light", "label": "market light", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "Review 1: the no-plugin result more convincingly completes the market through readable market and vendor signage, richer stall dressing and stronger architectural finishing. the plugin result satisfies the core spatial and construction criteria, but its missing signage and simpler secondary details make it feel less finished. Neither shows a decisive support or route failure. Review 2: the no-plugin result wins through clearer vendor identification, readable signage and more varied visible prop dressing. the plugin result provides coherent construction and usable circulation, but missing visible signs weakens completion of the brief. Neither sheet fully proves all reverse surfaces." }, { "id": "night_market_courtyard--claude-sonnet-5-5--r01", "task": "night_market_courtyard", "taskTitle": "Complete low-poly night-market courtyard", "model": "claude-sonnet-5-5", "modelTitle": "Sonnet 5.5", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 2, "rawReview": { "taskId": "night_market_courtyard", "repetition": 1, "baselineAutomatedScore": 100, "candidateAutomatedScore": 100, "hardGates": { "baseline": true, "candidate": true }, "visualWinnerVotes": [ "vanilla", "vanilla" ], "criterionMajorities": [ { "criterionId": "market_stalls", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_construction", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "market_route", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_support", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "market_palette", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_light", "baselinePasses": 2, "candidatePasses": 1, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "market_stalls", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_construction", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "market_route", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "market_support", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "market_palette", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7, "materialsAndReadability": 8, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 7, "lightingAndPresentation": 7.5, "finalStageCompleteness": 7.5, "multiviewConsistency": 8 }, "plugin": { "taskFidelity": 6.5, "silhouetteAndProportion": 6, "constructionPlausibility": 5, "craftsmanshipAndDetail": 6, "materialsAndReadability": 6.5, "surfaceFinishAndShading": 6, "materialTextureQuality": 6, "lightingAndPresentation": 6.5, "finalStageCompleteness": 5.5, "multiviewConsistency": 6 } }, "result": { "winner": "A", "confidence": 0.88, "scores": { "A": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 7.5, "craftsmanshipAndDetail": 7, "materialsAndReadability": 8, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 7, "lightingAndPresentation": 7.5, "finalStageCompleteness": 7.5, "multiviewConsistency": 8 }, "B": { "taskFidelity": 6.5, "silhouetteAndProportion": 6, "constructionPlausibility": 5, "craftsmanshipAndDetail": 6, "materialsAndReadability": 6.5, "surfaceFinishAndShading": 6, "materialTextureQuality": 6, "lightingAndPresentation": 6.5, "finalStageCompleteness": 5.5, "multiviewConsistency": 6 } }, "criterionResults": [ { "criterionId": "market_stalls", "A": "pass", "B": "pass", "evidenceA": "Separate red food, blue pottery and green flower stalls contain recognizable vendor inventory.", "evidenceB": "Food trays, pottery displays and a densely stocked flower counter distinguish three vendors." }, { "criterionId": "market_construction", "A": "pass", "B": "fail", "evidenceA": "Stalls have coherent posts, counters, shelves and continuous canopies; courtyard walls and paving look finished.", "evidenceB": "Foreground and reverse stall views expose irregular projecting canopy strips and awkward diagonal framing that read as unfinished assembly." }, { "criterionId": "market_route", "A": "pass", "B": "pass", "evidenceA": "The entrance connects to a broad paved center, with seating and vendors arranged along the perimeter.", "evidenceB": "A connected central paved area remains usable despite bulky foreground stall framing; perimeter benches leave circulation space." }, { "criterionId": "market_support", "A": "pass", "B": "fail", "evidenceA": "Inventory rests on counters and shelves, crates sit on paving, and lantern cables visibly terminate at facade brackets.", "evidenceB": "Several separated canopy boards project without clearly readable support or convincing connections to the stall frame." }, { "criterionId": "market_palette", "A": "pass", "B": "pass", "evidenceA": "Red, blue and green canopies establish vendor identity within restrained plaster, wood and stone colors.", "evidenceB": "Colored awnings and recognizable inventory distinguish vendors, although strong amber lighting reduces material separation." }, { "criterionId": "market_light", "A": "pass", "B": "pass", "evidenceA": "Warm lanterns and windows contrast with cool surroundings, and all vendor zones remain legible.", "evidenceB": "Warm lantern and window lighting keeps the counters, flowers and courtyard readable despite heavier shadows." } ], "majorDefects": { "A": [ "Food inventory and signage are comparatively sparse and schematic.", "Flat surfaces and simple furniture limit the sense of final refinement." ], "B": [ "Fragmented canopy construction and ambiguous board supports weaken the reverse views.", "Foreground framing crowds the presentation and obscures stall relationships.", "Strong amber shading compresses material distinctions." ] }, "rationale": "A presents a more coherent finished courtyard, with continuous stall roofs, credible supports and clearly organized circulation. B has appealing flower detail and atmosphere, but its exposed canopy assembly substantially weakens construction plausibility and completeness." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 6, "silhouetteAndProportion": 5, "constructionPlausibility": 4, "craftsmanshipAndDetail": 5, "materialsAndReadability": 6, "surfaceFinishAndShading": 6, "materialTextureQuality": 5, "lightingAndPresentation": 5, "finalStageCompleteness": 5, "multiviewConsistency": 6 }, "vanilla": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "result": { "winner": "B", "confidence": 0.89, "scores": { "A": { "taskFidelity": 6, "silhouetteAndProportion": 5, "constructionPlausibility": 4, "craftsmanshipAndDetail": 5, "materialsAndReadability": 6, "surfaceFinishAndShading": 6, "materialTextureQuality": 5, "lightingAndPresentation": 5, "finalStageCompleteness": 5, "multiviewConsistency": 6 }, "B": { "taskFidelity": 8, "silhouetteAndProportion": 8, "constructionPlausibility": 8, "craftsmanshipAndDetail": 7, "materialsAndReadability": 8, "surfaceFinishAndShading": 8, "materialTextureQuality": 7, "lightingAndPresentation": 8, "finalStageCompleteness": 8, "multiviewConsistency": 8 } }, "criterionResults": [ { "criterionId": "market_stalls", "A": "pass", "B": "pass", "evidenceA": "Food, pottery and flower displays visibly distinguish three stocked stalls.", "evidenceB": "Three separate canopy stalls contain food displays, pottery shelves and flowers." }, { "criterionId": "market_construction", "A": "fail", "B": "pass", "evidenceA": "Central and foreground stall structures have fragmented, sharply projecting canopy shapes and exposed diagonal assemblies that read as unfinished.", "evidenceB": "Posts, counters, shelving, canopies and courtyard walls form coherent finished structures across the supplied views." }, { "criterionId": "market_route", "A": "pass", "B": "pass", "evidenceA": "Connected paved space remains visible around the central structures, with seating along the facade.", "evidenceB": "The overview clearly shows an entrance connected to a broad central aisle, with stalls and appropriately scaled seating along its edges." }, { "criterionId": "market_support", "A": "unclear", "B": "pass", "evidenceA": "Flower pots and background inventory have counters, but cropped foreground assemblies and overlapping angular parts prevent verification of every support.", "evidenceB": "Inventory rests on counters or shelves; crates and furniture meet the paving, and lantern cables visibly terminate at poles or wall brackets." }, { "criterionId": "market_palette", "A": "pass", "B": "pass", "evidenceA": "Red, teal and green stall accents distinguish vendors within a consistent warm timber palette.", "evidenceB": "Red, blue and green canopies identify vendors, while restrained plaster, timber and paving colors unify the courtyard." }, { "criterionId": "market_light", "A": "fail", "B": "pass", "evidenceA": "Strong amber illumination and deeply shaded stall interiors reduce prop separation; cool night illumination is barely evident.", "evidenceB": "Cool blue surroundings balance warm lanterns and windows, with inventory, seating and circulation remaining legible." } ], "majorDefects": { "A": [ "Fragmented central and foreground canopy silhouettes weaken construction readability.", "Tight cropping and overlapping structures obscure support relationships and courtyard organization.", "Heavy amber lighting reduces material separation.", "Signs are pictorial or indistinct rather than clearly readable." ], "B": [ "Food inventory remains comparatively sparse and primitively shaped.", "Vendor signs use simple symbols rather than clearly readable lettering.", "Broad wall and counter surfaces have limited secondary detail." ] }, "rationale": "B presents a more complete, coherent courtyard with clearer circulation, credible stall construction and better visibility of supported inventory. A has appealing color and flowers, but fragmented foreground structures, obstructive framing and amber-heavy lighting substantially weaken its finish." } } ] }, "resourcePolicy": { "cpuHardCapPercent": 20, "priority": "BelowNormal", "concurrency": 1, "requestedBlenderThreads": 2, "guidanceFile": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-gpu-denoise-guidance.md", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4" }, "sharedResourceGuidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "limitation": "One generation per condition, medium effort, 30-minute generation limit. Two counterbalanced Astra reviews. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 2, "candidate": 0, "tie": 0 }, "vanilla": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 485, "triangles": 14944, "images": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "5a824bf72b8196f56791560091279b9efbac85fa8a08331c93d12c8f3b11872a", "reverse": "ccd09e3e21114344ac79b15dc517f8776ec0d38de3a80317fffc6fed02550744", "detail": "ffe1e2529dcd06a84a3a77f973a7ee689e661eb8e23bfc2446534e1018c6a696" }, "imageDimensions": { "hero": [ 1536, 864 ], "reverse": [ 1536, 864 ], "detail": [ 1536, 864 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "9638ab570aa43c8aa5916038d1250cf766d80e10f11c772a893da3c95fea0ac1", "reverse": "815b877f59eda29e2df193d2b6b479bb408c5411ceda71b7e2a8b24f47b8796c", "detail": "d72b005350c15dfccdde570a2b0b235f69a2f185b6c460ed180fbd0813e8a5c4" }, "originalImageDimensions": { "hero": [ 512, 288 ], "reverse": [ 512, 288 ], "detail": [ 512, 288 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\night_market_courtyard--claude-sonnet-5-5--vanilla\\night_market_courtyard-r01\\asset.blend", "sourceSha256": "403c9457644eca5a187fd1d5d50824e42586c4201c0695663fd48eb917c5396a", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "NightMarketCourtyard", "engine": "BLENDER_EEVEE", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 22, "type": "PERSP", "location": [ 0, -11.5, 7.599999904632568 ], "clipStart": 0.10000000149011612, "clipEnd": 100 }, { "name": "SceneReverse", "lens": 19, "type": "PERSP", "location": [ 0.6000000238418579, 4.699999809265137, 4.400000095367432 ], "clipStart": 0.10000000149011612, "clipEnd": 100 }, { "name": "SceneDetail", "lens": 24, "type": "PERSP", "location": [ 0.30000001192092896, -0.20000000298023224, 2 ], "clipStart": 0.10000000149011612, "clipEnd": 100 } ], "lights": [ { "name": "GateLight_left", "type": "POINT", "energy": 60, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "GateLight_right", "type": "POINT", "energy": 60, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "FoodStallLight", "type": "POINT", "energy": 150, "color": [ 1, 0.5, 0.20000000298023224 ], "hidden": false }, { "name": "CeramicsStallLight", "type": "POINT", "energy": 150, "color": [ 1, 0.7799999713897705, 0.5 ], "hidden": false }, { "name": "FlowerStallLight", "type": "POINT", "energy": 150, "color": [ 1, 0.6499999761581421, 0.6000000238418579 ], "hidden": false }, { "name": "table_a_light", "type": "POINT", "energy": 25, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "table_b_light", "type": "POINT", "energy": 25, "color": [ 1, 0.6000000238418579, 0.30000001192092896 ], "hidden": false }, { "name": "StringLight_0_3", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "StringLight_0_12", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "StringLight_0_21", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "StringLight_1_3", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "StringLight_1_12", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "StringLight_1_21", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "StringLight_2_3", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "StringLight_2_12", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "StringLight_2_21", "type": "POINT", "energy": 90, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "MoonSun", "type": "SUN", "energy": 0.800000011920929, "color": [ 0.550000011920929, 0.6499999761581421, 1 ], "hidden": false }, { "name": "CoolSkyFill", "type": "AREA", "energy": 400, "color": [ 0.44999998807907104, 0.550000011920929, 1 ], "hidden": false } ], "volumeMaterials": [], "world": "NightSky", "colorManagement": { "viewTransform": "AgX", "look": "None", "exposure": 0.30000001192092896, "gamma": 1 }, "resolution": [ 1280, 720, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-sonnet-5-5--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 1.436, "sha256": "5a824bf72b8196f56791560091279b9efbac85fa8a08331c93d12c8f3b11872a" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-sonnet-5-5--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 0.844, "sha256": "ccd09e3e21114344ac79b15dc517f8776ec0d38de3a80317fffc6fed02550744" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-sonnet-5-5--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 0.821, "sha256": "ffe1e2529dcd06a84a3a77f973a7ee689e661eb8e23bfc2446534e1018c6a696" } ], "device": { "requested": "OPTIX", "effective": "engine-managed", "devices": [] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "engine-managed", "denoiser": null, "quality": null, "prefilter": null, "gpu": null, "decisionReason": "Render engine is not Cycles.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 864 ], "samples": null, "timeLimitSeconds": null, "denoise": null, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\night_market_courtyard--claude-sonnet-5-5--vanilla\\night_market_courtyard-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "64f358e74dad211e5322d1cb231a6d3dba0fb61c64071df2806dad9443a588cc", "asset.blend": "403c9457644eca5a187fd1d5d50824e42586c4201c0695663fd48eb917c5396a", "asset.glb": "0dc1accfcaf165b1e4e7eddaef8e183ea3eef2b0fdc793d205bb953dfcfd80fc" }, "executionMode": "baseline", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2daae6f279a726c0e7cad74a138b8c87c9da2123f5c4dab94e082a1f423c1ea4", "agentVersion": "2.1.288 (Claude Code)", "codexTransport": null, "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 7 }, "plugin": { "score": 100, "rawScore": 100, "hardGate": true, "rawHardGate": true, "seconds": 448, "triangles": 14916, "images": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "2d49898b54eee5438fe5e4c47c214f823c730f27086bdef644f047503446ac40", "reverse": "e48f702dc9e7b8a20a664a211f060be86059e9e26eaf96f8c43db0b7a6509c65", "detail": "67dad05f259de3d82467a10c8a0163c04b997a9589f23dedff3b1bf64a10390b" }, "imageDimensions": { "hero": [ 1536, 864 ], "reverse": [ 1536, 864 ], "detail": [ 1536, 864 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/plugin/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/plugin/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/night_market_courtyard--claude-sonnet-5-5--r01/plugin/original/detail.png" }, "originalImageHashes": { "hero": "852a944e1bbcb6a1a471cc9d68854130823522186c94bfa01bd76639975fdd61", "reverse": "af73940aae2f8a7b2f08d1a2210fc4251cfd6eb48c85424bd1ebf2be44c4f684", "detail": "9bedced8dbfdc4e09c8296426b1d1449614920488298df11710eaf74bf5fecf0" }, "originalImageDimensions": { "hero": [ 512, 288 ], "reverse": [ 512, 288 ], "detail": [ 512, 288 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\night_market_courtyard--claude-sonnet-5-5--plugin\\night_market_courtyard-r01\\asset.blend", "sourceSha256": "aebffc649a7dbd523555e17600888285c551f3bfe12c50616acfbee0e801385a", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "NightMarketCourtyard", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 16, "type": "PERSP", "location": [ -0.5, -4.599999904632568, 3.950000047683716 ], "clipStart": 0.10000000149011612, "clipEnd": 100 }, { "name": "SceneReverse", "lens": 17, "type": "PERSP", "location": [ 0.30000001192092896, 4.5, 3.950000047683716 ], "clipStart": 0.10000000149011612, "clipEnd": 100 }, { "name": "SceneDetail", "lens": 26, "type": "PERSP", "location": [ -1.2000000476837158, 0.30000001192092896, 1.7000000476837158 ], "clipStart": 0.10000000149011612, "clipEnd": 100 } ], "lights": [ { "name": "Moon_CoolKey", "type": "SUN", "energy": 0.30000001192092896, "color": [ 0.44999998807907104, 0.6000000238418579, 1 ], "hidden": false }, { "name": "CoolSkyFill", "type": "AREA", "energy": 40, "color": [ 0.4000000059604645, 0.550000011920929, 1 ], "hidden": false }, { "name": "Warm_FoodStall", "type": "POINT", "energy": 150, "color": [ 1, 0.44999998807907104, 0.15000000596046448 ], "hidden": false }, { "name": "Warm_CeramicsStall", "type": "POINT", "energy": 150, "color": [ 1, 0.6000000238418579, 0.2800000011920929 ], "hidden": false }, { "name": "Warm_FlowerStall", "type": "POINT", "energy": 140, "color": [ 1, 0.6000000238418579, 0.3499999940395355 ], "hidden": false }, { "name": "Warm_Seating", "type": "POINT", "energy": 130, "color": [ 1, 0.550000011920929, 0.2199999988079071 ], "hidden": false }, { "name": "Warm_RouteCenter", "type": "POINT", "energy": 150, "color": [ 1, 0.550000011920929, 0.2199999988079071 ], "hidden": false }, { "name": "Warm_RouteWest", "type": "POINT", "energy": 90, "color": [ 1, 0.550000011920929, 0.2199999988079071 ], "hidden": false }, { "name": "Warm_RouteEast", "type": "POINT", "energy": 90, "color": [ 1, 0.550000011920929, 0.2199999988079071 ], "hidden": false } ], "volumeMaterials": [], "world": "NightSky", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": -0.4000000059604645, "gamma": 1 }, "resolution": [ 1280, 720, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-sonnet-5-5--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 4.566, "sha256": "2d49898b54eee5438fe5e4c47c214f823c730f27086bdef644f047503446ac40" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-sonnet-5-5--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 4.31, "sha256": "e48f702dc9e7b8a20a664a211f060be86059e9e26eaf96f8c43db0b7a6509c65" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\night_market_courtyard--claude-sonnet-5-5--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 4.668, "sha256": "67dad05f259de3d82467a10c8a0163c04b997a9589f23dedff3b1bf64a10390b" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 864 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\night_market_courtyard--claude-sonnet-5-5--plugin\\night_market_courtyard-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "937ae68ebab704526850711c4ad8599820988de3d06d4f2d52c3c60c39fb0e56", "asset.blend": "aebffc649a7dbd523555e17600888285c551f3bfe12c50616acfbee0e801385a", "asset.glb": "517da427b3402340bc353368b2a3e778f13d8636d377f0881545309686421a94" }, "executionMode": "skills", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "2daae6f279a726c0e7cad74a138b8c87c9da2123f5c4dab94e082a1f423c1ea4", "agentVersion": "2.1.288 (Claude Code)", "codexTransport": null, "modelRequested": "claude-sonnet-5-5", "modelsObserved": [ "claude-sonnet-5-5" ], "rawScorerVersion": 7 }, "criteria": [ { "id": "market_stalls", "label": "market stalls", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "market_construction", "label": "market construction", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 2, "unclear": 0 } }, { "id": "market_route", "label": "market route", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "market_support", "label": "market support", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 1, "unclear": 1 } }, { "id": "market_palette", "label": "market palette", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "market_light", "label": "market light", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 1, "fail": 1, "unclear": 0 } } ], "note": "Review 1: the no-plugin result presents a more coherent finished courtyard, with continuous stall roofs, credible supports and clearly organized circulation. the plugin result has appealing flower detail and atmosphere, but its exposed canopy assembly substantially weakens construction plausibility and completeness. Review 2: the no-plugin result presents a more complete, coherent courtyard with clearer circulation, credible stall construction and better visibility of supported inventory. the plugin result has appealing color and flowers, but fragmented foreground structures, obstructive framing and amber-heavy lighting substantially weaken its finish." }, { "id": "coastal_cafe_holdout--gpt-6-astra--r01", "task": "coastal_cafe_holdout", "taskTitle": "Decorated coastal cafe holdout", "model": "gpt-6-astra", "modelTitle": "GPT 6 Astra", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 2, "rawReview": { "taskId": "coastal_cafe_holdout", "repetition": 1, "baselineAutomatedScore": 94, "candidateAutomatedScore": 91.5, "hardGates": { "baseline": false, "candidate": false }, "visualWinnerVotes": [ "vanilla", "vanilla" ], "criterionMajorities": [ { "criterionId": "cafe_service", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "cafe_seating", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_integration", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "cafe_glass", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_reverse", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_composition", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "cafe_service", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "cafe_seating", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_integration", "baselinePasses": 2, "candidatePasses": 0, "judgeCount": 2 }, { "criterionId": "cafe_glass", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_reverse", "baselinePasses": 0, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.6, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.8, "multiviewConsistency": 7 }, "plugin": { "taskFidelity": 6.5, "silhouetteAndProportion": 7.4, "constructionPlausibility": 5.9, "craftsmanshipAndDetail": 7, "materialsAndReadability": 7.2, "surfaceFinishAndShading": 7.6, "materialTextureQuality": 6.7, "lightingAndPresentation": 6.2, "finalStageCompleteness": 6.6, "multiviewConsistency": 8.5 } }, "result": { "winner": "A", "confidence": 0.94, "scores": { "A": { "taskFidelity": 8.8, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.8, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.6, "lightingAndPresentation": 8.7, "finalStageCompleteness": 8.8, "multiviewConsistency": 7 }, "B": { "taskFidelity": 6.5, "silhouetteAndProportion": 7.4, "constructionPlausibility": 5.9, "craftsmanshipAndDetail": 7, "materialsAndReadability": 7.2, "surfaceFinishAndShading": 7.6, "materialTextureQuality": 6.7, "lightingAndPresentation": 6.2, "finalStageCompleteness": 6.6, "multiviewConsistency": 8.5 } }, "criterionResults": [ { "criterionId": "cafe_service", "A": "pass", "B": "fail", "evidenceA": "Finished tiled counter, espresso machine, stocked pastry case and visibly printed menu signage.", "evidenceB": "Counter, espresso machine and stocked display are present, but the upright menu-like cards have no readable markings." }, { "criterionId": "cafe_seating", "A": "pass", "B": "pass", "evidenceA": "Multiple furnished table groups include chairs, banquette seating and tableware.", "evidenceB": "Two round tables have chairs and tableware, with additional banquette seating visible." }, { "criterionId": "cafe_integration", "A": "pass", "B": "fail", "evidenceA": "Pendant cords meet the ceiling; furniture and dishes have credible supports, with visible circulation space.", "evidenceB": "Furniture and tableware are supported, but pendant cords terminate in floating caps without a ceiling or supporting structure." }, { "criterionId": "cafe_glass", "A": "pass", "B": "pass", "evidenceA": "The framed glass pastry case clearly exposes shelves and individual baked goods.", "evidenceB": "Display glazing permits visible shelves and pastries in the perspective and service-facing views." }, { "criterionId": "cafe_reverse", "A": "unclear", "B": "pass", "evidenceA": "Secondary interior areas look finished, but the supplied views do not establish reverse-side finish.", "evidenceB": "Rear and side elevations retain finished wood panels, upholstery, window framing and consistent materials." }, { "criterionId": "cafe_composition", "A": "pass", "B": "fail", "evidenceA": "Warm window illumination and coordinated coastal colors distinguish the service counter, seating and decorated secondary areas.", "evidenceB": "The layout is readable, especially from above, but neutral studio lighting and a black backdrop do not establish morning light." } ], "majorDefects": { "A": [ "Interior perspective views leave reverse surfaces and the full overhead circulation layout unverified." ], "B": [ "Pendant lights lack visible architectural supports.", "Menu cards lack readable menu detail.", "Open architectural shell and studio presentation weaken the impression of a complete, morning-lit cafe." ] }, "rationale": "A delivers a more complete coastal interior, with purposeful decoration, richer material detail, readable signage and supported lighting. B offers stronger all-around inspection and consistent finishes, but floating pendants, blank signage and absent morning atmosphere reduce task fidelity and completeness." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "vanilla", "scoresByMode": { "plugin": { "taskFidelity": 6.5, "silhouetteAndProportion": 7.5, "constructionPlausibility": 6.5, "craftsmanshipAndDetail": 7, "materialsAndReadability": 7, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 6.5, "lightingAndPresentation": 6, "finalStageCompleteness": 6.5, "multiviewConsistency": 8 }, "vanilla": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.5, "lightingAndPresentation": 8, "finalStageCompleteness": 8.5, "multiviewConsistency": 6.5 } }, "result": { "winner": "B", "confidence": 0.91, "scores": { "A": { "taskFidelity": 6.5, "silhouetteAndProportion": 7.5, "constructionPlausibility": 6.5, "craftsmanshipAndDetail": 7, "materialsAndReadability": 7, "surfaceFinishAndShading": 7.5, "materialTextureQuality": 6.5, "lightingAndPresentation": 6, "finalStageCompleteness": 6.5, "multiviewConsistency": 8 }, "B": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8.5, "craftsmanshipAndDetail": 8.5, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 8.5, "lightingAndPresentation": 8, "finalStageCompleteness": 8.5, "multiviewConsistency": 6.5 } }, "criterionResults": [ { "criterionId": "cafe_service", "A": "unclear", "B": "pass", "evidenceA": "Counter, espresso machine and pastry case are visible; small upright panels do not read clearly as completed menu signs.", "evidenceB": "Finished counter, espresso machine, readable pastry case and wall menu signage are visible." }, { "criterionId": "cafe_seating", "A": "pass", "B": "pass", "evidenceA": "Two table groups have chairs, adjacent bench seating where appropriate, and usable floor space.", "evidenceB": "Both seating groups have finished tables, chairs or upholstered seating, and tabletop settings." }, { "criterionId": "cafe_integration", "A": "fail", "B": "pass", "evidenceA": "Furniture and dishes have supports, but pendant stems terminate in empty space without a visible ceiling or suspension framework.", "evidenceB": "Pendants attach to the ceiling, dishes rest on tables, and the visible circulation between service and seating remains clear." }, { "criterionId": "cafe_glass", "A": "pass", "B": "pass", "evidenceA": "The display enclosure permits visible trays and pastries behind its glazing.", "evidenceB": "Transparent framed glazing clearly reveals pastries and separate display shelves." }, { "criterionId": "cafe_reverse", "A": "pass", "B": "unclear", "evidenceA": "Rear and side views retain finished wood panels, upholstery, framing and coordinated trim.", "evidenceB": "The secondary self-service area is finished, but the supplied interior perspectives do not establish reverse-side finish." }, { "criterionId": "cafe_composition", "A": "fail", "B": "pass", "evidenceA": "Service and seating are distinguishable, but gray studio illumination does not establish readable morning light.", "evidenceB": "Warm daylight, window views and coherent interior framing distinguish the service counter from the furnished seating areas." } ], "majorDefects": { "A": [ "Pendant suspension lacks visible architectural support.", "Menu signage is not readable enough to establish completion.", "Studio presentation lacks the requested morning atmosphere." ], "B": [ "Three cropped interior perspectives omit explicit reverse and top views, limiting verification of hidden surfaces and overall circulation." ] }, "rationale": "B delivers a more complete cafe through finished architecture, supported lighting, readable signage, textured materials and purposeful decoration. A is coherent across more views, but its unsupported pendants, uncertain menus and studio lighting weaken fidelity and completeness. B's limited viewpoint coverage remains a significant evidence gap." } } ] }, "resourcePolicy": { "cpuHardCapPercent": 20, "priority": "BelowNormal", "concurrency": 1, "requestedBlenderThreads": 2, "guidanceFile": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-gpu-denoise-guidance.md", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4" }, "sharedResourceGuidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "limitation": "One generation per condition, medium effort, 30-minute generation limit. Two counterbalanced Astra reviews. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 2, "candidate": 0, "tie": 0 }, "vanilla": { "score": 96, "rawScore": 94, "hardGate": false, "rawHardGate": false, "seconds": 1161, "triangles": 306820, "images": { "hero": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "12d89d0a6c641498b327e6a30e4373412ce2e74f91f6c7cfd6811152d2cfa747", "reverse": "593c18aed32d4487fe7ffc1a2c7576df645c54ccc2b592d42ab8eedffbcd75b9", "detail": "a488b36899852cd542750d34ccec8459c4e31f29bdaafe41dee17d28638389ee" }, "imageDimensions": { "hero": [ 1536, 1152 ], "reverse": [ 1536, 1152 ], "detail": [ 1536, 1152 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "8277e7de1b2fd907dacaea1e44573a0a3b987b5efd613ef305dd0c0fa240ffb5", "reverse": "3e857f9ffcca42da1afb3529627afaba2040124b73a8cc3e529dfb67ded5cbe9", "detail": "58f3226a21dce8f26a3be93f37fc3a0d28cc718e140edc0c4030812c46a2ffce" }, "originalImageDimensions": { "hero": [ 512, 384 ], "reverse": [ 512, 384 ], "detail": [ 512, 384 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\coastal_cafe_holdout--gpt-6-astra--vanilla\\coastal_cafe_holdout-r01\\asset.blend", "sourceSha256": "1d456c4473897c1f23827a376d4d870d64c5ac09f2bea49df5035f331e5688d8", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 20.5, "type": "PERSP", "location": [ -3.1600000858306885, -2.799999952316284, 2.5799999237060547 ], "clipStart": 0.05000000074505806, "clipEnd": 300 }, { "name": "SceneReverse", "lens": 20.5, "type": "PERSP", "location": [ 3.119999885559082, 2.2200000286102295, 2.4800000190734863 ], "clipStart": 0.05000000074505806, "clipEnd": 300 }, { "name": "SceneDetail", "lens": 43, "type": "PERSP", "location": [ 0.15000000596046448, -0.05999999865889549, 1.850000023841858 ], "clipStart": 0.05000000074505806, "clipEnd": 300 } ], "lights": [ { "name": "Pastry case concealed shelf lighting", "type": "AREA", "energy": 6, "color": [ 1, 0.8399999737739563, 0.6200000047683716 ], "hidden": false }, { "name": "Counter pendant one downlight", "type": "AREA", "energy": 45, "color": [ 1, 0.7900000214576721, 0.5400000214576721 ], "hidden": false }, { "name": "Counter pendant two downlight", "type": "AREA", "energy": 45, "color": [ 1, 0.7900000214576721, 0.5400000214576721 ], "hidden": false }, { "name": "Round table pendant downlight", "type": "AREA", "energy": 45, "color": [ 1, 0.7900000214576721, 0.5400000214576721 ], "hidden": false }, { "name": "Window table pendant downlight", "type": "AREA", "energy": 45, "color": [ 1, 0.7900000214576721, 0.5400000214576721 ], "hidden": false }, { "name": "Morning through sea windows", "type": "AREA", "energy": 1100, "color": [ 1, 0.8799999952316284, 0.699999988079071 ], "hidden": false }, { "name": "Soft sky across glass", "type": "AREA", "energy": 420, "color": [ 0.7200000286102295, 0.8600000143051147, 1 ], "hidden": false }, { "name": "Entrance diffuse fill", "type": "AREA", "energy": 340, "color": [ 0.8600000143051147, 0.9200000166893005, 1 ], "hidden": false }, { "name": "Ceiling bounced morning", "type": "AREA", "energy": 120, "color": [ 1, 0.9100000262260437, 0.7900000214576721 ], "hidden": false }, { "name": "Low morning sun", "type": "SUN", "energy": 1.350000023841858, "color": [ 1, 0.8899999856948853, 0.7200000286102295 ], "hidden": false } ], "volumeMaterials": [], "world": "World", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": -0.6499999761581421, "gamma": 1 }, "resolution": [ 1440, 1080, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6-astra--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 10.694, "sha256": "12d89d0a6c641498b327e6a30e4373412ce2e74f91f6c7cfd6811152d2cfa747" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6-astra--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 10.775, "sha256": "593c18aed32d4487fe7ffc1a2c7576df645c54ccc2b592d42ab8eedffbcd75b9" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6-astra--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 10.029, "sha256": "a488b36899852cd542750d34ccec8459c4e31f29bdaafe41dee17d28638389ee" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1152 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "4114 invalid vertices, degenerate faces, or zero-length edges" } ], "rawFailedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "4114 invalid vertices, degenerate faces, or zero-length edges" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "220.000m maximum extent; limit 18m" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\coastal_cafe_holdout--gpt-6-astra--vanilla\\coastal_cafe_holdout-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "cc4dfaeefd914c44cae040ef44022a042062b467e538ca5adb118e4823d2e49d", "asset.blend": "1d456c4473897c1f23827a376d4d870d64c5ac09f2bea49df5035f331e5688d8" }, "executionMode": "baseline", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "cb1ff5abce0cae0b37a6279c116ddca24c4123fa63f735bcf6ed38b13a2f3880", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6-astra", "modelsObserved": null, "rawScorerVersion": 7 }, "plugin": { "score": 93.5, "rawScore": 91.5, "hardGate": false, "rawHardGate": false, "seconds": 1685, "triangles": 286448, "images": { "hero": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "6f59f830f424d31d287de7b58efe7b2ec0d472f7c19cf2f4e060076fcaf7f3c9", "reverse": "f8e2300ee63f4c071a191a5bf3b6b6fad2ff9ff4a7bf11892c048ee48a76aa23", "detail": "b118ddf3c10619147e2b19117e0d151504f171e9d34d51bf1bd1591ca37c7b89" }, "imageDimensions": { "hero": [ 1536, 1152 ], "reverse": [ 1536, 1152 ], "detail": [ 1536, 1152 ] }, "originalImages": { "perspective": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/original/perspective.png", "front": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/original/front.png", "back": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/original/back.png", "left": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/original/left.png", "right": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/original/right.png", "top": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6-astra--r01/plugin/original/top.png" }, "originalImageHashes": { "perspective": "e5f0f070d614d86e98f04898cdbe3de96c451e264d1c799421f0f77b0dfa707a", "front": "dcab8a3f3f72191e526e5c5e7017f515555f8c25ced4cf3be7e3c482dc71c370", "back": "aa75f5af49ed897e6fe7421f72bc3c26059a401f93a6625014126f83fb89e4ec", "left": "3fc3c0cac5ce1038d3b6e057c2b6acd762f656a2a20bc6f032151ee4b0807fae", "right": "e4732c7cb206a88a1fa7a079c64e6e36ada98646b9172cf42a484f561ed68887", "top": "c7af2161a9a11729ec6360fa9394fd4bb91a190f9109bb65d3c7704a9cc0134a" }, "originalImageDimensions": { "perspective": [ 512, 512 ], "front": [ 512, 512 ], "back": [ 512, 512 ], "left": [ 512, 512 ], "right": [ 512, 512 ], "top": [ 512, 512 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\coastal_cafe_holdout--gpt-6-astra--plugin\\coastal_cafe_holdout-r01\\asset.blend", "sourceSha256": "18ea93431aad20efa09c2fb87edc3af83a53e9e5cc67d825951ea205599ef303", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 23, "type": "PERSP", "location": [ -0.4000000059604645, -5.099999904632568, 2.3499999046325684 ], "clipStart": 0.10000000149011612, "clipEnd": 250 }, { "name": "SceneReverse", "lens": 21, "type": "PERSP", "location": [ 2.549999952316284, 2.630000114440918, 2.0799999237060547 ], "clipStart": 0.10000000149011612, "clipEnd": 250 }, { "name": "SceneDetail", "lens": 37, "type": "PERSP", "location": [ 0.6499999761581421, -1.9500000476837158, 1.850000023841858 ], "clipStart": 0.10000000149011612, "clipEnd": 250 } ], "lights": [ { "name": "Morning through seaside glazing", "type": "AREA", "energy": 1700, "color": [ 1, 0.8700000047683716, 0.6899999976158142 ], "hidden": false }, { "name": "Entrance sky fill", "type": "AREA", "energy": 520, "color": [ 0.7699999809265137, 0.8700000047683716, 1 ], "hidden": false }, { "name": "Soft ceiling bounce", "type": "AREA", "energy": 180, "color": [ 1, 0.8999999761581421, 0.7699999809265137 ], "hidden": false }, { "name": "Low morning sun", "type": "SUN", "energy": 1.25, "color": [ 1, 1, 1 ], "hidden": false }, { "name": "Service pendant A pool", "type": "AREA", "energy": 45, "color": [ 1, 0.7900000214576721, 0.5099999904632568 ], "hidden": false }, { "name": "Service pendant B pool", "type": "AREA", "energy": 45, "color": [ 1, 0.7900000214576721, 0.5099999904632568 ], "hidden": false }, { "name": "Window pendant pool", "type": "AREA", "energy": 45, "color": [ 1, 0.7900000214576721, 0.5099999904632568 ], "hidden": false }, { "name": "Reading pendant pool", "type": "AREA", "energy": 45, "color": [ 1, 0.7900000214576721, 0.5099999904632568 ], "hidden": false } ], "volumeMaterials": [], "world": "Soft coastal morning", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": 0, "gamma": 1 }, "resolution": [ 1440, 1080, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6-astra--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 12.291, "sha256": "6f59f830f424d31d287de7b58efe7b2ec0d472f7c19cf2f4e060076fcaf7f3c9" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6-astra--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 10.28, "sha256": "f8e2300ee63f4c071a191a5bf3b6b6fad2ff9ff4a7bf11892c048ee48a76aa23" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6-astra--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 8.734, "sha256": "b118ddf3c10619147e2b19117e0d151504f171e9d34d51bf1bd1591ca37c7b89" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1152 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "authored_render_complete", "passed": false, "earned": 0, "possible": 4, "detail": "All required authored-camera images rendered with dependencies present" }, { "id": "authored_render_dependencies", "passed": false, "earned": 0, "possible": 2.5, "detail": "Authored render has no missing image/library dependencies" }, { "id": "authored_camera_evidence", "passed": false, "earned": 0, "possible": 0, "detail": "All contracted scene cameras must render with available dependencies; this does not certify composition or finish" } ], "rawFailedChecks": [ { "id": "authored_render_complete", "passed": false, "earned": 0, "possible": 4, "detail": "All required authored-camera images rendered with dependencies present" }, { "id": "authored_render_dependencies", "passed": false, "earned": 0, "possible": 2.5, "detail": "Authored render has no missing image/library dependencies" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "160.000m maximum extent; limit 18m" }, { "id": "authored_camera_evidence", "passed": false, "earned": 0, "possible": 0, "detail": "All contracted scene cameras must render with available dependencies; this does not certify composition or finish" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\coastal_cafe_holdout--gpt-6-astra--plugin\\coastal_cafe_holdout-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "d5482e6e0d6a3c2fa9e215d45b2aaf631059422e108ff1bf604ac217aba2628b", "asset.blend": "18ea93431aad20efa09c2fb87edc3af83a53e9e5cc67d825951ea205599ef303" }, "executionMode": "skills", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "cb1ff5abce0cae0b37a6279c116ddca24c4123fa63f735bcf6ed38b13a2f3880", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6-astra", "modelsObserved": null, "rawScorerVersion": 7 }, "criteria": [ { "id": "cafe_service", "label": "cafe service", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 1, "unclear": 1 } }, { "id": "cafe_seating", "label": "cafe seating", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cafe_integration", "label": "cafe integration", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 2, "unclear": 0 } }, { "id": "cafe_glass", "label": "cafe glass", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cafe_reverse", "label": "cafe reverse", "vanilla": { "pass": 0, "fail": 0, "unclear": 2 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cafe_composition", "label": "cafe composition", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 0, "fail": 2, "unclear": 0 } } ], "note": "Review 1: the no-plugin result delivers a more complete coastal interior, with purposeful decoration, richer material detail, readable signage and supported lighting. the plugin result offers stronger all-around inspection and consistent finishes, but floating pendants, blank signage and absent morning atmosphere reduce task fidelity and completeness. Review 2: the no-plugin result delivers a more complete cafe through finished architecture, supported lighting, readable signage, textured materials and purposeful decoration. the plugin result is coherent across more views, but its unsupported pendants, uncertain menus and studio lighting weaken fidelity and completeness. the no-plugin result's limited viewpoint coverage remains a significant evidence gap." }, { "id": "coastal_cafe_holdout--gpt-6.1-sol--r01", "task": "coastal_cafe_holdout", "taskTitle": "Decorated coastal cafe holdout", "model": "gpt-6.1-sol", "modelTitle": "GPT 6.1 Sol", "repetition": 1, "cohort": "October 2026 scenes and game characters", "judgeCount": 2, "rawReview": { "taskId": "coastal_cafe_holdout", "repetition": 1, "baselineAutomatedScore": 94, "candidateAutomatedScore": 98, "hardGates": { "baseline": false, "candidate": false }, "visualWinnerVotes": [ "vanilla", "plugin" ], "criterionMajorities": [ { "criterionId": "cafe_service", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_seating", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_integration", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_glass", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_reverse", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_composition", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "criticalCriterionMajorities": [ { "criterionId": "cafe_service", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_seating", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_integration", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_glass", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 }, { "criterionId": "cafe_reverse", "baselinePasses": 2, "candidatePasses": 2, "judgeCount": 2 } ], "verifiedScores": null, "judgeResults": [ { "mapping": { "A": "vanilla", "B": "plugin" }, "decodedWinner": "vanilla", "scoresByMode": { "vanilla": { "taskFidelity": 9, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.4, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 8.7 }, "plugin": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.8, "lightingAndPresentation": 8.3, "finalStageCompleteness": 8.3, "multiviewConsistency": 8.4 } }, "result": { "winner": "A", "confidence": 0.83, "scores": { "A": { "taskFidelity": 9, "silhouetteAndProportion": 8.6, "constructionPlausibility": 8.4, "craftsmanshipAndDetail": 8.8, "materialsAndReadability": 8.7, "surfaceFinishAndShading": 8.8, "materialTextureQuality": 8.7, "lightingAndPresentation": 8.5, "finalStageCompleteness": 9, "multiviewConsistency": 8.7 }, "B": { "taskFidelity": 8.5, "silhouetteAndProportion": 8, "constructionPlausibility": 8.2, "craftsmanshipAndDetail": 8, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.5, "materialTextureQuality": 7.8, "lightingAndPresentation": 8.3, "finalStageCompleteness": 8.3, "multiviewConsistency": 8.4 } }, "criterionResults": [ { "criterionId": "cafe_service", "A": "pass", "B": "pass", "evidenceA": "Finished tiled counter, espresso machine, stocked pastry case and framed menu signs are visible.", "evidenceB": "Finished teal counter, espresso machine, pastry case and labeled menu boards establish a complete service area." }, { "criterionId": "cafe_seating", "A": "pass", "B": "pass", "evidenceA": "Multiple seating groups combine furnished tables, chairs and upholstered benches.", "evidenceB": "Bench seating and separate tables have chairs, dishes and usable table surfaces." }, { "criterionId": "cafe_integration", "A": "pass", "B": "pass", "evidenceA": "Tableware rests on tables, pendants have suspension cords, and circulation remains visible between seating and service.", "evidenceB": "Tables have substantial pedestal supports, pendants are suspended, and the central floor provides clear access." }, { "criterionId": "cafe_glass", "A": "pass", "B": "pass", "evidenceA": "The close view shows transparent display panels with readable pastries, trays and internal shelves.", "evidenceB": "Pastries and shelves remain visible through the service counter display glazing." }, { "criterionId": "cafe_reverse", "A": "pass", "B": "pass", "evidenceA": "Secondary seating, side cabinetry, wall trim and the display's alternate angle maintain the same material finish.", "evidenceB": "Secondary cabinetry, counter equipment and bench close-up retain consistent wood, teal and upholstery finishes." }, { "criterionId": "cafe_composition", "A": "pass", "B": "pass", "evidenceA": "Soft window illumination keeps the tiled service zone and distributed seating distinct, although pale signs lose contrast.", "evidenceB": "Warm daylight and teal accents distinguish service and seating, with broad open circulation clearly visible." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "A wins through richer material texture, finer furniture detailing and a more convincingly stocked transparent display. B satisfies the brief, but its oversized bench cushions, simpler pastry forms and repetitive trim make the finish less refined. Both maintain coherent secondary areas and clear circulation." } }, { "mapping": { "A": "plugin", "B": "vanilla" }, "decodedWinner": "plugin", "scoresByMode": { "plugin": { "taskFidelity": 8.7, "silhouetteAndProportion": 8.4, "constructionPlausibility": 8.3, "craftsmanshipAndDetail": 8.7, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.4, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.7, "multiviewConsistency": 8.5 }, "vanilla": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.2, "constructionPlausibility": 8.3, "craftsmanshipAndDetail": 8.1, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 8, "lightingAndPresentation": 8.2, "finalStageCompleteness": 8.3, "multiviewConsistency": 8.4 } }, "result": { "winner": "A", "confidence": 0.7, "scores": { "A": { "taskFidelity": 8.7, "silhouetteAndProportion": 8.4, "constructionPlausibility": 8.3, "craftsmanshipAndDetail": 8.7, "materialsAndReadability": 8.5, "surfaceFinishAndShading": 8.7, "materialTextureQuality": 8.4, "lightingAndPresentation": 8.5, "finalStageCompleteness": 8.7, "multiviewConsistency": 8.5 }, "B": { "taskFidelity": 8.5, "silhouetteAndProportion": 8.2, "constructionPlausibility": 8.3, "craftsmanshipAndDetail": 8.1, "materialsAndReadability": 8.4, "surfaceFinishAndShading": 8.2, "materialTextureQuality": 8, "lightingAndPresentation": 8.2, "finalStageCompleteness": 8.3, "multiviewConsistency": 8.4 } }, "criterionResults": [ { "criterionId": "cafe_service", "A": "pass", "B": "pass", "evidenceA": "Finished trimmed counter, detailed espresso machine, stocked glass display and contrasting menu boards are visible.", "evidenceB": "Tiled service counter, espresso machine, stocked pastry case and wall menu signs are visible." }, { "criterionId": "cafe_seating", "A": "pass", "B": "pass", "evidenceA": "Banquette and freestanding seating include supported tables, chairs and tableware.", "evidenceB": "Separate seating groups contain benches or chairs, round tables and table settings." }, { "criterionId": "cafe_integration", "A": "pass", "B": "pass", "evidenceA": "Tables have visible bases, dishes rest on surfaces, pendants have suspension cords and the central aisle remains open.", "evidenceB": "Furniture legs, bench frames and pendant cords provide credible support; circulation remains visible between seating and service." }, { "criterionId": "cafe_glass", "A": "pass", "B": "pass", "evidenceA": "Tinted display glazing reveals shelves and individual pastries.", "evidenceB": "The close display view clearly reveals pastries and trays through reflective transparent panels." }, { "criterionId": "cafe_reverse", "A": "pass", "B": "pass", "evidenceA": "Secondary cabinet, wall art, planting and seating retain the coordinated wood, cream and teal finish.", "evidenceB": "Secondary seating, storage cabinet, wall trim and decorations retain consistent finished materials." }, { "criterionId": "cafe_composition", "A": "pass", "B": "pass", "evidenceA": "Warm daylight and soft shadows maintain clear separation between service counter, seating and open circulation.", "evidenceB": "Soft daylight keeps both zones readable, although the bright doorway and pale surfaces reduce contrast." } ], "majorDefects": { "A": [], "B": [] }, "rationale": "A narrowly wins through more refined upholstery, counter detailing, material contrast and warm lighting. B provides a particularly readable pastry display and complete layout, but its slab-like bench upholstery and more uniform pale shading feel less polished. Both visibly satisfy the core requirements." } } ] }, "resourcePolicy": { "cpuHardCapPercent": 20, "priority": "BelowNormal", "concurrency": 1, "requestedBlenderThreads": 2, "guidanceFile": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-gpu-denoise-guidance.md", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4" }, "sharedResourceGuidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "limitation": "One generation per condition, medium effort, 30-minute generation limit. Two counterbalanced Astra reviews. Resource limits changed between cohorts; time is not a latency benchmark. Pinned skills and scripts; this is not an MCP ablation. Authored cameras and lighting are part of the result; cameras may differ between attempts. The slider is a side-by-side comparison, not an aligned geometry difference. Technical scores use scorer 9; original scores and gates are retained. Any replacement evaluator images render the same hashed saved source and are recorded separately.", "votes": { "baseline": 1, "candidate": 1, "tie": 0 }, "vanilla": { "score": 96, "rawScore": 94, "hardGate": false, "rawHardGate": false, "seconds": 1154, "triangles": 321572, "images": { "hero": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/vanilla/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/vanilla/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/vanilla/preview/detail.png" }, "imageHashes": { "hero": "729b440e3ed27966ac1affe65425ae261cf13e3c192e21d056f94a27d984bcc6", "reverse": "e612efd260356068c57ac63be7f52493f39663f9b3667a4b82b9ec942b4ea887", "detail": "76e455bd30f5fbbb459648ebd94e70585c78d5d48331aa64970fabbd8290f2cc" }, "imageDimensions": { "hero": [ 1536, 1152 ], "reverse": [ 1536, 1152 ], "detail": [ 1536, 1152 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/vanilla/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/vanilla/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/vanilla/original/detail.png" }, "originalImageHashes": { "hero": "bf2ba8813911f31c06a1bffe8b029768a477f6b64cfbbf07b045f36a92430d3a", "reverse": "a665b7e7c0aede02874c9d1cf31c0e377ba467ccbcedb54da150c883944eca6f", "detail": "332bbf777989b8e11c2954dc1bfb66f4edf682db9c825467e25c60bf3bce63b2" }, "originalImageDimensions": { "hero": [ 512, 384 ], "reverse": [ 512, 384 ], "detail": [ 512, 384 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\coastal_cafe_holdout--gpt-6.1-sol--vanilla\\coastal_cafe_holdout-r01\\asset.blend", "sourceSha256": "f120c146c997caee79ae7317bdef7229246b52556268d3e3634a3e8fb278c611", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 24, "type": "PERSP", "location": [ -0.25, -5.099999904632568, 2.75 ], "clipStart": 0.05000000074505806, "clipEnd": 300 }, { "name": "SceneReverse", "lens": 22, "type": "PERSP", "location": [ 3.1500000953674316, 2.390000104904175, 2.5 ], "clipStart": 0.05000000074505806, "clipEnd": 300 }, { "name": "SceneDetail", "lens": 46, "type": "PERSP", "location": [ -0.20000000298023224, -2.3299999237060547, 1.909999966621399 ], "clipStart": 0.05000000074505806, "clipEnd": 300 } ], "lights": [ { "name": "Service pendant A illumination", "type": "AREA", "energy": 65, "color": [ 1, 0.800000011920929, 0.5799999833106995 ], "hidden": false }, { "name": "Service pendant B illumination", "type": "AREA", "energy": 65, "color": [ 1, 0.800000011920929, 0.5799999833106995 ], "hidden": false }, { "name": "Window table pendant illumination", "type": "AREA", "energy": 65, "color": [ 1, 0.800000011920929, 0.5799999833106995 ], "hidden": false }, { "name": "Reading table pendant illumination", "type": "AREA", "energy": 65, "color": [ 1, 0.800000011920929, 0.5799999833106995 ], "hidden": false }, { "name": "Morning window key", "type": "AREA", "energy": 950, "color": [ 1, 0.8799999952316284, 0.7099999785423279 ], "hidden": false }, { "name": "Entrance soft skylight", "type": "AREA", "energy": 350, "color": [ 0.7400000095367432, 0.8399999737739563, 1 ], "hidden": false }, { "name": "Ceiling diffuse interior bounce", "type": "AREA", "energy": 210, "color": [ 1, 0.8899999856948853, 0.75 ], "hidden": false }, { "name": "Morning angled sunlight", "type": "SUN", "energy": 0.800000011920929, "color": [ 1, 1, 1 ], "hidden": false } ], "volumeMaterials": [], "world": "Morning marine atmosphere", "colorManagement": { "viewTransform": "AgX", "look": "None", "exposure": 0, "gamma": 1 }, "resolution": [ 1600, 1200, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6.1-sol--r01\\vanilla\\camera-01_frame-000001.png", "durationSeconds": 10.452, "sha256": "729b440e3ed27966ac1affe65425ae261cf13e3c192e21d056f94a27d984bcc6" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6.1-sol--r01\\vanilla\\camera-02_frame-000001.png", "durationSeconds": 10.674, "sha256": "e612efd260356068c57ac63be7f52493f39663f9b3667a4b82b9ec942b4ea887" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6.1-sol--r01\\vanilla\\camera-03_frame-000001.png", "durationSeconds": 9.642, "sha256": "76e455bd30f5fbbb459648ebd94e70585c78d5d48331aa64970fabbd8290f2cc" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1152 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "7105 invalid vertices, degenerate faces, or zero-length edges" } ], "rawFailedChecks": [ { "id": "invalid_geometry", "passed": false, "earned": 0, "possible": 4, "detail": "7105 invalid vertices, degenerate faces, or zero-length edges" }, { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "240.000m maximum extent; limit 18m" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\coastal_cafe_holdout--gpt-6.1-sol--vanilla\\coastal_cafe_holdout-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "d059f117b247cb71785a3af92d7c5a27c615b614c89959575d664d17d21304d3", "asset.blend": "f120c146c997caee79ae7317bdef7229246b52556268d3e3634a3e8fb278c611" }, "executionMode": "baseline", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "skillFingerprint": null, "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "cb1ff5abce0cae0b37a6279c116ddca24c4123fa63f735bcf6ed38b13a2f3880", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 7 }, "plugin": { "score": 100, "rawScore": 98, "hardGate": true, "rawHardGate": false, "seconds": 1117, "triangles": 229117, "images": { "hero": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/plugin/preview/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/plugin/preview/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/plugin/preview/detail.png" }, "imageHashes": { "hero": "3aa9dccec1171aae67e720ea6781c5447af3eccf051eb5292c4d162285b0ce2f", "reverse": "a797033a01d4bc06d14c9c98d058dd8c652212eddc4bd7d20a4791cb7ee70a93", "detail": "e9e6df5f882f5d4da559f1b868767897a9a07fb6d72b50b4f042bb99ab2d49f0" }, "imageDimensions": { "hero": [ 1536, 1097 ], "reverse": [ 1536, 1097 ], "detail": [ 1536, 1097 ] }, "originalImages": { "hero": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/plugin/original/hero.png", "reverse": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/plugin/original/reverse.png", "detail": "benchmarks/gallery-matrix/scene-character/coastal_cafe_holdout--gpt-6.1-sol--r01/plugin/original/detail.png" }, "originalImageHashes": { "hero": "ccc9a7e279394b6263a2c64238d652e52ea3d061b7f6329198ec7d16480742e3", "reverse": "d35b2688dd44ac130d1e13f4f8b3d6072a82169b06cef6015c4d447c524d2e14", "detail": "2c8cc8cd5eda1dec6c291fd88cb214009cd444bd7fd577ef7276423bfaf58cd1" }, "originalImageDimensions": { "hero": [ 512, 366 ], "reverse": [ 512, 366 ], "detail": [ 512, 366 ] }, "preview": { "schemaVersion": 1, "status": "complete", "source": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\coastal_cafe_holdout--gpt-6.1-sol--plugin\\coastal_cafe_holdout-r01\\asset.blend", "sourceSha256": "de4adfda0f873c80f961dac22a67c9dca5e9fa16f2d71f698b067691eb916908", "blenderVersion": "5.2.2 LTS", "blenderBuildHash": "d13f752e3b9c", "preflight": { "scene": "Scene", "engine": "CYCLES", "activeCamera": "SceneHero", "cameras": [ { "name": "SceneHero", "lens": 20, "type": "PERSP", "location": [ -3.0999999046325684, -2.7300000190734863, 1.9600000381469727 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneReverse", "lens": 23, "type": "PERSP", "location": [ 3.0799999237060547, 2.0399999618530273, 1.899999976158142 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 }, { "name": "SceneDetail", "lens": 44, "type": "PERSP", "location": [ 0.23000000417232513, -2.549999952316284, 1.6799999475479126 ], "clipStart": 0.05000000074505806, "clipEnd": 1000 } ], "lights": [ { "name": "Pendant warm pool 0", "type": "AREA", "energy": 45, "color": [ 1, 0.7699999809265137, 0.49000000953674316 ], "hidden": false }, { "name": "Pendant warm pool 1", "type": "AREA", "energy": 45, "color": [ 1, 0.7699999809265137, 0.49000000953674316 ], "hidden": false }, { "name": "Pendant warm pool 2", "type": "AREA", "energy": 45, "color": [ 1, 0.7699999809265137, 0.49000000953674316 ], "hidden": false }, { "name": "Pendant warm pool 3", "type": "AREA", "energy": 45, "color": [ 1, 0.7699999809265137, 0.49000000953674316 ], "hidden": false }, { "name": "Broad seaside morning light", "type": "AREA", "energy": 550, "color": [ 1, 0.8799999952316284, 0.699999988079071 ], "hidden": false }, { "name": "North window bounce", "type": "AREA", "energy": 350, "color": [ 0.699999988079071, 0.8399999737739563, 1 ], "hidden": false }, { "name": "Entrance soft fill", "type": "AREA", "energy": 140, "color": [ 1, 0.9100000262260437, 0.800000011920929 ], "hidden": false }, { "name": "Morning sun", "type": "SUN", "energy": 1.100000023841858, "color": [ 1, 1, 1 ], "hidden": false } ], "volumeMaterials": [], "world": "World", "colorManagement": { "viewTransform": "AgX", "look": "AgX - Medium High Contrast", "exposure": 0.30000001192092896, "gamma": 1 }, "resolution": [ 1400, 1000, 100 ], "frame": 1, "images": [], "missingDependencies": [], "warnings": [] }, "renders": [ { "camera": "SceneHero", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6.1-sol--r01\\plugin\\camera-01_frame-000001.png", "durationSeconds": 8.08, "sha256": "3aa9dccec1171aae67e720ea6781c5447af3eccf051eb5292c4d162285b0ce2f" }, { "camera": "SceneReverse", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6.1-sol--r01\\plugin\\camera-02_frame-000001.png", "durationSeconds": 8.101, "sha256": "a797033a01d4bc06d14c9c98d058dd8c652212eddc4bd7d20a4791cb7ee70a93" }, { "camera": "SceneDetail", "frame": 1, "path": "D:\\CodexArtifacts\\blender-gallery-hd-2026-10-04\\renders\\coastal_cafe_holdout--gpt-6.1-sol--r01\\plugin\\camera-03_frame-000001.png", "durationSeconds": 7.529, "sha256": "e9e6df5f882f5d4da559f1b868767897a9a07fb6d72b50b4f042bb99ab2d49f0" } ], "device": { "requested": "OPTIX", "effective": "OPTIX", "devices": [ "NVIDIA GeForce RTX 3060 Ti" ] }, "denoise": { "requestedPolicy": "final", "effectivePolicy": "final", "denoiser": "OPENIMAGEDENOISE", "quality": "HIGH", "prefilter": "ACCURATE", "gpu": true, "inputPasses": "RGB_ALBEDO_NORMAL", "renderDenoisingEnabled": true, "decisionReason": "Final uses supported OIDN GPU denoising.", "compositorDenoiseNodes": [] }, "mutedFileOutputs": [], "effective": { "resolution": [ 1536, 1097 ], "samples": 64, "timeLimitSeconds": 120, "denoise": true, "compositing": true, "sequencer": false, "border": false }, "description": "Previews preserve authored cameras, lighting, materials and compositing, with a 1536 px resolution cap. Engine and sample settings are recorded in the data.", "hiddenStagingObjects": [] }, "evidenceUnavailable": false, "failedChecks": [], "rawFailedChecks": [ { "id": "maximum_extent", "passed": false, "earned": 0, "possible": 2, "detail": "70.000m maximum extent; limit 18m" } ], "failure": null, "scorerVersion": 9, "rescoreEvaluatorFingerprint": "93ea2514609cb7ebff588c986fb22afa4b2ee171c0cbbe5c0f800a11acd72852", "scoreCorrection": "Scorer 9 checks finite bounds for render-only scenes and recognizes ordinary food ingredient names. Geometry and export gates are unchanged. Raw scores are retained. Any repaired evaluator evidence uses the same saved source hash.", "evidenceDirectory": "C:\\Users\\ghost\\Desktop\\Coding\\Blender\\benchmarks\\scene-character-2026-10-03-capped\\coastal_cafe_holdout--gpt-6.1-sol--plugin\\coastal_cafe_holdout-r01\\evidence", "evaluatorEvidenceRepaired": false, "artifactHashes": { "create_asset.py": "c75cb54a392db41d92f11b015c5f7b590d39458b958649914ffe7b91d33eb11d", "asset.blend": "de4adfda0f873c80f961dac22a67c9dca5e9fa16f2d71f698b067691eb916908" }, "executionMode": "skills", "guidanceHash": "8c9c030f01034f60417002c434c33a4020f7b121b41078c3c0d70fab7d9d77c4", "skillFingerprint": "6c2f75fb8f57c83625f23e2a254f146bcd2b45569b5a920e0d5aefc05a114891", "evaluatorFingerprint": "1b64272bb062873b43b334905bdc417515480052bc8b881848fc9ea90f594bf8", "generationTaskFingerprint": "cb1ff5abce0cae0b37a6279c116ddca24c4123fa63f735bcf6ed38b13a2f3880", "agentVersion": "codex-cli 0.160.0", "codexTransport": "http", "modelRequested": "gpt-6.1-sol", "modelsObserved": null, "rawScorerVersion": 7 }, "criteria": [ { "id": "cafe_service", "label": "cafe service", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cafe_seating", "label": "cafe seating", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cafe_integration", "label": "cafe integration", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cafe_glass", "label": "cafe glass", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cafe_reverse", "label": "cafe reverse", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } }, { "id": "cafe_composition", "label": "cafe composition", "vanilla": { "pass": 2, "fail": 0, "unclear": 0 }, "plugin": { "pass": 2, "fail": 0, "unclear": 0 } } ], "note": "Review 1: the no-plugin result wins through richer material texture, finer furniture detailing and a more convincingly stocked transparent display. the plugin result satisfies the brief, but its oversized bench cushions, simpler pastry forms and repetitive trim make the finish less refined. Both maintain coherent secondary areas and clear circulation. Review 2: the plugin result narrowly wins through more refined upholstery, counter detailing, material contrast and warm lighting. the no-plugin result provides a particularly readable pastry display and complete layout, but its slab-like bench upholstery and more uniform pale shading feel less polished. Both visibly satisfy the core requirements." } ], "framing": "Preview resolution and evidence presentation vary by task; exact dimensions are shown in the viewer.", "limitation": "Separate historical cohorts; one paired generation per task/model except the explicitly curated historical lantern. No aggregate capability claim.", "selection": { "historicalLantern": { "model": "gpt-6-sol", "repetition": 3, "requestedBy": "gallery owner", "earlierAttempts": "https://github.com/ifBars/blender-agent-studio/releases/tag/gallery-hd-2026-10-02" } } }