{"schema_version":1,"title":"3D Asset Library measured benchmarks","status":"Historical measurements; no general efficacy or quality-parity claim.","study_verdicts":{"TF1":"NO_WIN","TF2":"UNVERIFIABLE"},"formal_acceptance":"UNVERIFIED","sources":{"tf1_report":{"original_path":"tfeature/REPORT.md","sha256":"c3607f0c30c6739e9bf614ec2e4935a29477f359b7fcd6a9feabe86fbd568aca","public_url":"/static/benchmark-sources/tf1_report.md","public_sha256":"5c83e6cc46eace2bc4dcccf103bf326adbb99a10a3f99ad871d57586bea21dc9","representation":"report copy with model nicknames expanded to recorded identifiers; measurements unchanged"},"tf1_results":{"original_path":"tfeature/results.json","sha256":"aaa7f815061b91115ccc7547ed5d7323ec945793429da5997f9d4d17865c64f0","public_url":"/static/benchmark-sources/tf1_results.json","public_sha256":"b3e23d9456aee743cd4aebefad0e519079d4fba375f6acc3d2544919acbe5902","representation":"aggregate extract; original full file hash differs"},"tf2_report":{"original_path":"tfeature/tf2/REPORT.md","sha256":"f67ff3459b77a6af8b6466b7b9004391b58024aebe8792bfd969995fb7d39bbf","public_url":"/static/benchmark-sources/tf2_report.md","public_sha256":"7956818a1f998188d4b3823fad96ab3f42a0fdea44d233fb8ccec8ad4e473768","representation":"report copy with model nicknames expanded to recorded identifiers; measurements unchanged"},"tf2_results":{"original_path":"tfeature/tf2/results.json","sha256":"76109faeb0e0bdb930b5a8a2bf5eb166a74d067f988ed7440ef94f3ca54fd62a","public_url":"/static/benchmark-sources/tf2_results.json","public_sha256":"9cd1e2b9dc101f6c6b450003e70675c7b4beae4a95206c18cc362d4112aca56f","representation":"aggregate extract; original full file hash differs"}},"results":[{"id":"tf1-sol","metric":"mean audit defects per asset; dimensions percent; weighted tokens per finished asset; total wall seconds per arm","study":"TF1","subset":"STANDARD","arms":["FAM-S","RAW-S"],"values":{"library":0.16666666666666666,"raw":1.0416666666666667,"ratio":6.250000000000001,"library_dimensions":100.0,"raw_dimensions":100.0,"library_tokens":117075.75,"raw_tokens":67798.60833333332,"token_ratio":1.7268164181836085,"library_agent_seconds":6926.8,"raw_agent_seconds":4366.1,"library_queue_seconds":12520.3998336792,"raw_queue_seconds":27580.0},"n":24,"test":"Exact paired one-sided sign flips; ties retained.","p":0.03125,"ci":null,"date":"2026-10-22T05:12:04Z","model_identifiers":{"FAM-S":["gpt-6.1-sol"],"RAW-S":["gpt-6.1-sol"]},"model_identity":"Requested CLI identifiers; provider-returned identity UNVERIFIED.","conditions":"Paired briefs, library versus raw Blender code using the same requested model. Dimension tolerance is 3%. Weighted tokens = uncached input + 0.1 cached input + output, all attempts / finished assets.","limitations":["TF1 did not meet its full pre-registered win criteria; formal acceptance has not been verified.","Historical import and framing violations retained; same-model comparison under this harness, not general efficacy.","Queue-inclusive time includes different harness queue reacquisition; not an isolated speed comparison."],"source_report":"/static/benchmark-sources/tf1_report.md","provenance":{"values.library":{"source":"tf1_results","pointer":"/summaries/FAM-S/mean_defects"},"values.raw":{"source":"tf1_results","pointer":"/summaries/RAW-S/mean_defects"},"values.library_tokens":{"source":"tf1_results","pointer":"/summaries/FAM-S/tokens_per_finished_asset"},"values.raw_tokens":{"source":"tf1_results","pointer":"/summaries/RAW-S/tokens_per_finished_asset"},"values.library_agent_seconds":{"source":"tf1_results","pointer":"/summaries/FAM-S/agent_seconds"},"values.raw_agent_seconds":{"source":"tf1_results","pointer":"/summaries/RAW-S/agent_seconds"},"values.library_queue_seconds":{"source":"tf1_results","pointer":"/summaries/FAM-S/queue_inclusive_seconds"},"values.raw_queue_seconds":{"source":"tf1_results","pointer":"/summaries/RAW-S/queue_inclusive_seconds"},"n":{"source":"tf1_results","pointer":"/comparisons/FAM-S_vs_RAW-S/defect_test/n"},"p":{"source":"tf1_results","pointer":"/comparisons/FAM-S_vs_RAW-S/defect_test/p"},"values.token_ratio":{"source":"tf1_results","pointer":"/comparisons/FAM-S_vs_RAW-S/tokens_ratio"}},"derived":{"values.ratio":{"operation":"divide","inputs":["values.raw","values.library"]},"values.library_dimensions":{"operation":"percent","source":{"source":"tf1_results","pointer":"/summaries/FAM-S/mean_adherence"}},"values.raw_dimensions":{"operation":"percent","source":{"source":"tf1_results","pointer":"/summaries/RAW-S/mean_adherence"}}}},{"id":"tf1-luna","metric":"mean audit defects per asset; dimensions percent; weighted tokens per finished asset; total wall seconds per arm","study":"TF1","subset":"STANDARD","arms":["FAM-L","RAW-L"],"values":{"library":1.7916666666666667,"raw":2.75,"ratio":1.5348837209302324,"library_dimensions":95.83333333333334,"raw_dimensions":94.44444444444444,"library_tokens":127955.14166666666,"raw_tokens":61674.49166666667,"token_ratio":2.0746849825407288,"library_agent_seconds":7436.2,"raw_agent_seconds":2341.5,"library_queue_seconds":13841.34836268425,"raw_queue_seconds":23588.0},"n":24,"test":"Exact paired one-sided sign flips; ties retained.","p":0.284912109375,"ci":null,"date":"2026-10-22T05:12:04Z","model_identifiers":{"FAM-L":["gpt-6-luna"],"RAW-L":["gpt-6-luna"]},"model_identity":"Requested CLI identifiers; provider-returned identity UNVERIFIED.","conditions":"Paired briefs, library versus raw Blender code using the same requested model. Dimension tolerance is 3%. Weighted tokens = uncached input + 0.1 cached input + output, all attempts / finished assets.","limitations":["TF1 did not meet its full pre-registered win criteria; formal acceptance has not been verified.","Historical import and framing violations retained; same-model comparison under this harness, not general efficacy.","Queue-inclusive time includes different harness queue reacquisition; not an isolated speed comparison.","gpt-6-luna defect difference is not significant."],"source_report":"/static/benchmark-sources/tf1_report.md","provenance":{"values.library":{"source":"tf1_results","pointer":"/summaries/FAM-L/mean_defects"},"values.raw":{"source":"tf1_results","pointer":"/summaries/RAW-L/mean_defects"},"values.library_tokens":{"source":"tf1_results","pointer":"/summaries/FAM-L/tokens_per_finished_asset"},"values.raw_tokens":{"source":"tf1_results","pointer":"/summaries/RAW-L/tokens_per_finished_asset"},"values.library_agent_seconds":{"source":"tf1_results","pointer":"/summaries/FAM-L/agent_seconds"},"values.raw_agent_seconds":{"source":"tf1_results","pointer":"/summaries/RAW-L/agent_seconds"},"values.library_queue_seconds":{"source":"tf1_results","pointer":"/summaries/FAM-L/queue_inclusive_seconds"},"values.raw_queue_seconds":{"source":"tf1_results","pointer":"/summaries/RAW-L/queue_inclusive_seconds"},"n":{"source":"tf1_results","pointer":"/comparisons/FAM-L_vs_RAW-L/defect_test/n"},"p":{"source":"tf1_results","pointer":"/comparisons/FAM-L_vs_RAW-L/defect_test/p"},"values.token_ratio":{"source":"tf1_results","pointer":"/comparisons/FAM-L_vs_RAW-L/tokens_ratio"}},"derived":{"values.ratio":{"operation":"divide","inputs":["values.raw","values.library"]},"values.library_dimensions":{"operation":"percent","source":{"source":"tf1_results","pointer":"/summaries/FAM-L/mean_adherence"}},"values.raw_dimensions":{"operation":"percent","source":{"source":"tf1_results","pointer":"/summaries/RAW-L/mean_adherence"}}}},{"id":"tf2-luna-defects","metric":"mean audit defects per attempted STANDARD brief","study":"TF2","subset":"STANDARD","arms":["FAM2-L","RAW-L"],"values":{"library":6.375,"raw":28.666666666666668,"ratio":4.496732026143791},"n":24,"test":"Descriptive aggregate; no significant paired defect test is established.","p":null,"ci":null,"date":"2026-10-22T23:50:51Z","model_identifiers":{"FAM2-L":["gpt-6-luna"],"RAW-L":["gpt-6-luna"]},"model_identity":"Requested CLI identifiers; provider-returned identity UNVERIFIED.","conditions":"Frozen external audit: floating, below_ground, clash/interpenetration, gross wrong_scale. Unbuilt completed assets score 36 defects.","limitations":["TF2 could not verify its primary outcome; missing audit/metering and retained protocol failures prevent primary qualification.","Secondary comparisons have no multiplicity correction; no quality parity or universal model benefit established.","RAW builds from an empty scene; library contains previously refined knowledge. Equal inference budget is not equal lifetime development cost.","RAW gpt-6-luna refusals and unbuilt penalties are instruction-handling outcomes, not evidence of geometric inability.","Tool-exposure, shortened admission windows and unpinned shared-dependency limitations are retained in the source report."],"source_report":"/static/benchmark-sources/tf2_report.md","provenance":{"values.library":{"source":"tf2_results","pointer":"/aggregates/STANDARD/FAM2-L/mean_defects"},"values.raw":{"source":"tf2_results","pointer":"/aggregates/STANDARD/RAW-L/mean_defects"},"n":{"source":"tf2_results","pointer":"/aggregates/STANDARD/FAM2-L/attempted"}},"derived":{"values.ratio":{"operation":"divide","inputs":["values.raw","values.library"]}}},{"id":"tf2-complex-pairwise","metric":"Blind visual preference, gpt-6-luna library vs raw gpt-6-luna","study":"TF2","subset":"COMPLEX","arms":["FAM2-L","RAW-L"],"values":{"mean":0.5},"n":10,"test":"paired brief-level percentile bootstrap; one-sided 95% lower bound","p":null,"ci":{"lower":0.1,"confidence":"paired brief-level percentile bootstrap; one-sided 95% lower bound","resamples":20000,"seed":20261022},"date":"2026-10-22T23:50:51Z","model_identifiers":{"FAM2-L":["gpt-6-luna"],"RAW-L":["gpt-6-luna"]},"model_identity":"Requested CLI identifiers; provider-returned identity UNVERIFIED.","conditions":"Both orders judged by gpt-6.1-sol; votes averaged within brief. Positive values favor library. Original refined reference shown as target for COMPLEX.","limitations":["TF2 could not verify its primary outcome; missing audit/metering and retained protocol failures prevent primary qualification.","Secondary comparisons have no multiplicity correction; no quality parity or universal model benefit established.","RAW builds from an empty scene; library contains previously refined knowledge. Equal inference budget is not equal lifetime development cost.","RAW gpt-6-luna refusals and unbuilt penalties are instruction-handling outcomes, not evidence of geometric inability.","Tool-exposure, shortened admission windows and unpinned shared-dependency limitations are retained in the source report.","Positive lower bound on this secondary set; distinct from the failed refinement endpoint."],"source_report":"/static/benchmark-sources/tf2_report.md","provenance":{"values.mean":{"source":"tf2_results","pointer":"/secondary/COMPLEX/FAM2-L_vs_RAW-L/pairwise/mean"},"n":{"source":"tf2_results","pointer":"/secondary/COMPLEX/FAM2-L_vs_RAW-L/pairwise/n"},"ci.lower":{"source":"tf2_results","pointer":"/secondary/COMPLEX/FAM2-L_vs_RAW-L/pairwise/lower"},"ci.confidence":{"source":"tf2_results","pointer":"/secondary/COMPLEX/FAM2-L_vs_RAW-L/pairwise/method"},"ci.resamples":{"source":"tf2_results","pointer":"/secondary/COMPLEX/FAM2-L_vs_RAW-L/pairwise/resamples"},"ci.seed":{"source":"tf2_results","pointer":"/secondary/COMPLEX/FAM2-L_vs_RAW-L/pairwise/seed"}}},{"id":"tf2-sol-preference","metric":"Blind visual preference, gpt-6-luna library vs raw gpt-6.1-sol","study":"TF2","subset":"STANDARD","arms":["FAM2-L","RAW-S"],"values":{"mean":-0.625},"n":24,"test":"paired brief-level percentile bootstrap; one-sided 95% lower bound","p":null,"ci":{"lower":-0.8333333333333334,"confidence":"paired brief-level percentile bootstrap; one-sided 95% lower bound","resamples":20000,"seed":20261022},"date":"2026-10-22T23:50:51Z","model_identifiers":{"FAM2-L":["gpt-6-luna"],"RAW-S":["gpt-6.1-sol"]},"model_identity":"Requested CLI identifiers; provider-returned identity UNVERIFIED.","conditions":"Both orders judged by gpt-6.1-sol; votes averaged within brief. Positive values favor library. Original refined reference shown as target for COMPLEX.","limitations":["TF2 could not verify its primary outcome; missing audit/metering and retained protocol failures prevent primary qualification.","Secondary comparisons have no multiplicity correction; no quality parity or universal model benefit established.","RAW builds from an empty scene; library contains previously refined knowledge. Equal inference budget is not equal lifetime development cost.","RAW gpt-6-luna refusals and unbuilt penalties are instruction-handling outcomes, not evidence of geometric inability.","Tool-exposure, shortened admission windows and unpinned shared-dependency limitations are retained in the source report.","Raw gpt-6.1-sol is preferred; no quality-parity claim."],"source_report":"/static/benchmark-sources/tf2_report.md","provenance":{"values.mean":{"source":"tf2_results","pointer":"/secondary/STANDARD/FAM2-L_vs_RAW-S/pairwise/mean"},"n":{"source":"tf2_results","pointer":"/secondary/STANDARD/FAM2-L_vs_RAW-S/pairwise/n"},"ci.lower":{"source":"tf2_results","pointer":"/secondary/STANDARD/FAM2-L_vs_RAW-S/pairwise/lower"},"ci.confidence":{"source":"tf2_results","pointer":"/secondary/STANDARD/FAM2-L_vs_RAW-S/pairwise/method"},"ci.resamples":{"source":"tf2_results","pointer":"/secondary/STANDARD/FAM2-L_vs_RAW-S/pairwise/resamples"},"ci.seed":{"source":"tf2_results","pointer":"/secondary/STANDARD/FAM2-L_vs_RAW-S/pairwise/seed"}}},{"id":"tf2-complex-cost","metric":"published list-price USD per finished COMPLEX asset","study":"TF2","subset":"COMPLEX","arms":["FAM2-L","RAW-S"],"values":{"library":0.024381820000000002,"raw":0.21964764444444443,"ratio":9.008664834882893,"library_finished":7,"raw_finished":9},"n":10,"test":"Descriptive list-price estimate; no invoice or significance test.","p":null,"ci":null,"date":"2026-10-22T23:50:51Z","model_identifiers":{"FAM2-L":["gpt-6-luna"],"RAW-S":["gpt-6.1-sol"]},"model_identity":"Requested CLI identifiers; provider-returned identity UNVERIFIED.","conditions":"Total attempted-arm token cost / finished assets. Published Standard short-context input, cached input and output rates retrieved 2026-10-22.","limitations":["TF2 could not verify its primary outcome; missing audit/metering and retained protocol failures prevent primary qualification.","Secondary comparisons have no multiplicity correction; no quality parity or universal model benefit established.","RAW builds from an empty scene; library contains previously refined knowledge. Equal inference budget is not equal lifetime development cost.","RAW gpt-6-luna refusals and unbuilt penalties are instruction-handling outcomes, not evidence of geometric inability.","Tool-exposure, shortened admission windows and unpinned shared-dependency limitations are retained in the source report.","Not an invoice or Codex subscription charge; excludes cache writes, tools, hosting and library development compute.","Service tier, backend identity and long-context billing unproven; STANDARD priced endpoint remains unavailable."],"source_report":"/static/benchmark-sources/tf2_report.md","provenance":{"values.library":{"source":"tf2_results","pointer":"/aggregates/COMPLEX/FAM2-L/published_cost_per_finished_usd"},"values.raw":{"source":"tf2_results","pointer":"/aggregates/COMPLEX/RAW-S/published_cost_per_finished_usd"},"values.library_finished":{"source":"tf2_results","pointer":"/aggregates/COMPLEX/FAM2-L/finished"},"values.raw_finished":{"source":"tf2_results","pointer":"/aggregates/COMPLEX/RAW-S/finished"},"n":{"source":"tf2_results","pointer":"/aggregates/COMPLEX/FAM2-L/attempted"}},"derived":{"values.ratio":{"operation":"divide","inputs":["values.raw","values.library"]}}}],"build_compute_cost":null,"build_compute_cost_note":"No build-compute figure is published in this section; token estimates exclude development and hosting."}