{ "aggregates": [ { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "glm-openrouter", "cost_per_success_usd": 0.146551, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "hermes", "median_cost_usd": 0.150947, "median_wall_seconds": 516.232, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 0.439653, "total_wall_seconds": 1730.516, "workload": "eventforge" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "glm-openrouter", "cost_per_success_usd": 0.05023, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "pft", "median_cost_usd": 0.050411, "median_wall_seconds": 265.688, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 0.150689, "total_wall_seconds": 1180.345, "workload": "eventforge" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "kimi-openrouter", "cost_per_success_usd": 0.802998, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "hermes", "median_cost_usd": 0.356091, "median_wall_seconds": 314.765, "runs": 3, "success_rate": 0.6667, "successes": 2, "total_cost_usd": 1.605997, "total_wall_seconds": 1792.318, "workload": "eventforge" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "kimi-openrouter", "cost_per_success_usd": 0.380092, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "pft", "median_cost_usd": 0.348651, "median_wall_seconds": 582.386, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 1.140275, "total_wall_seconds": 1571.644, "workload": "eventforge" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "glm-openrouter", "cost_per_success_usd": 0.341135, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "hermes", "median_cost_usd": 0.323866, "median_wall_seconds": 288.692, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 1.023405, "total_wall_seconds": 1063.555, "workload": "queuecraft" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "glm-openrouter", "cost_per_success_usd": 0.321662, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "pft", "median_cost_usd": 0.272193, "median_wall_seconds": 377.436, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 0.964986, "total_wall_seconds": 957.643, "workload": "queuecraft" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "glm-vercel", "cost_per_success_usd": 0.579976, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "hermes", "median_cost_usd": 0.572873, "median_wall_seconds": 304.641, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 1.739928, "total_wall_seconds": 790.371, "workload": "queuecraft" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "glm-vercel", "cost_per_success_usd": 0.349323, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "pft", "median_cost_usd": 0.359863, "median_wall_seconds": 158.72, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 1.04797, "total_wall_seconds": 476.914, "workload": "queuecraft" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "kimi-openrouter", "cost_per_success_usd": 1.601043, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "hermes", "median_cost_usd": 1.549447, "median_wall_seconds": 705.385, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 4.803129, "total_wall_seconds": 1844.472, "workload": "queuecraft" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "kimi-openrouter", "cost_per_success_usd": 0.305103, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "pft", "median_cost_usd": 0.34042, "median_wall_seconds": 143.329, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 0.91531, "total_wall_seconds": 435.786, "workload": "queuecraft" }, { "all_in_output_estimate_total_usd": null, "all_in_output_estimate_upper_bound_total_usd": null, "cell": "glm-openrouter", "cost_per_success_usd": 3.741205, "costed_runs": 3, "image_output_estimate_total_usd": null, "image_output_estimate_upper_bound_total_usd": null, "lane": "hermes", "median_cost_usd": 1.179643, "median_wall_seconds": 1456.798, "runs": 3, "success_rate": 0.3333, "successes": 1, "total_cost_usd": 3.741205, "total_wall_seconds": 5366.347, "workload": "visual_site" }, { "all_in_output_estimate_total_usd": 4.230076, "all_in_output_estimate_upper_bound_total_usd": 4.230076, "cell": "glm-openrouter", "cost_per_success_usd": 0.519692, "costed_runs": 3, "image_output_estimate_total_usd": 2.671, "image_output_estimate_upper_bound_total_usd": 2.671, "lane": "pft", "median_cost_usd": 0.374547, "median_wall_seconds": 1497.89, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 1.559076, "total_wall_seconds": 4554.221, "workload": "visual_site" }, { "all_in_output_estimate_total_usd": 9.549964, "all_in_output_estimate_upper_bound_total_usd": 9.549964, "cell": "kimi-openrouter", "cost_per_success_usd": 2.523321, "costed_runs": 3, "image_output_estimate_total_usd": 1.98, "image_output_estimate_upper_bound_total_usd": 1.98, "lane": "hermes", "median_cost_usd": 2.172302, "median_wall_seconds": 1840.704, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 7.569964, "total_wall_seconds": 5622.937, "workload": "visual_site" }, { "all_in_output_estimate_total_usd": 6.661211, "all_in_output_estimate_upper_bound_total_usd": 6.661211, "cell": "kimi-openrouter", "cost_per_success_usd": 1.74907, "costed_runs": 3, "image_output_estimate_total_usd": 1.414, "image_output_estimate_upper_bound_total_usd": 1.414, "lane": "pft", "median_cost_usd": 1.686337, "median_wall_seconds": 1546.911, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 5.247211, "total_wall_seconds": 4439.645, "workload": "visual_site" }, { "all_in_output_estimate_total_usd": 21.733316, "all_in_output_estimate_upper_bound_total_usd": 21.733316, "cell": "opus", "cost_per_success_usd": 10.055158, "costed_runs": 3, "image_output_estimate_total_usd": 1.623, "image_output_estimate_upper_bound_total_usd": 1.623, "lane": "cc", "median_cost_usd": 6.642805, "median_wall_seconds": 1689.07, "runs": 3, "success_rate": 0.6667, "successes": 2, "total_cost_usd": 20.110315, "total_wall_seconds": 5337.666, "workload": "visual_site" }, { "all_in_output_estimate_total_usd": 10.344594, "all_in_output_estimate_upper_bound_total_usd": 10.344594, "cell": "opus", "cost_per_success_usd": 2.891865, "costed_runs": 3, "image_output_estimate_total_usd": 1.669, "image_output_estimate_upper_bound_total_usd": 1.669, "lane": "pft", "median_cost_usd": 2.85552, "median_wall_seconds": 988.026, "runs": 3, "success_rate": 1.0, "successes": 3, "total_cost_usd": 8.675595, "total_wall_seconds": 3061.889, "workload": "visual_site" } ], "comparisons": [ { "cell": "glm-openrouter", "directional_speed_waves": [ true, true, true ], "opponent": "hermes", "opponent_successes": 3, "pft_all_in_output_estimate_advantage_x_total": null, "pft_all_in_output_estimate_savings_percent": null, "pft_cost_advantage_x_total": 2.9176, "pft_cost_savings_percent": 65.73, "pft_speedup_x_total": 1.4661, "pft_successes": 3, "speed_result": "consistent_pft", "workload": "eventforge" }, { "cell": "kimi-openrouter", "directional_speed_waves": [ false, true ], "opponent": "hermes", "opponent_successes": 2, "pft_all_in_output_estimate_advantage_x_total": null, "pft_all_in_output_estimate_savings_percent": null, "pft_cost_advantage_x_total": 1.4084, "pft_cost_savings_percent": 29.0, "pft_speedup_x_total": 1.1404, "pft_successes": 3, "speed_result": "mixed_or_incomplete", "workload": "eventforge" }, { "cell": "glm-openrouter", "directional_speed_waves": [ true, false, false ], "opponent": "hermes", "opponent_successes": 3, "pft_all_in_output_estimate_advantage_x_total": null, "pft_all_in_output_estimate_savings_percent": null, "pft_cost_advantage_x_total": 1.0605, "pft_cost_savings_percent": 5.71, "pft_speedup_x_total": 1.1106, "pft_successes": 3, "speed_result": "mixed_or_incomplete", "workload": "queuecraft" }, { "cell": "glm-vercel", "directional_speed_waves": [ true, true, true ], "opponent": "hermes", "opponent_successes": 3, "pft_all_in_output_estimate_advantage_x_total": null, "pft_all_in_output_estimate_savings_percent": null, "pft_cost_advantage_x_total": 1.6603, "pft_cost_savings_percent": 39.77, "pft_speedup_x_total": 1.6573, "pft_successes": 3, "speed_result": "consistent_pft", "workload": "queuecraft" }, { "cell": "kimi-openrouter", "directional_speed_waves": [ true, true, true ], "opponent": "hermes", "opponent_successes": 3, "pft_all_in_output_estimate_advantage_x_total": null, "pft_all_in_output_estimate_savings_percent": null, "pft_cost_advantage_x_total": 5.2475, "pft_cost_savings_percent": 80.94, "pft_speedup_x_total": 4.2325, "pft_successes": 3, "speed_result": "consistent_pft", "workload": "queuecraft" }, { "cell": "glm-openrouter", "directional_speed_waves": [ false ], "opponent": "hermes", "opponent_successes": 1, "pft_all_in_output_estimate_advantage_x_total": null, "pft_all_in_output_estimate_savings_percent": null, "pft_cost_advantage_x_total": 2.3996, "pft_cost_savings_percent": 58.33, "pft_speedup_x_total": 1.1783, "pft_successes": 3, "speed_result": "mixed_or_incomplete", "workload": "visual_site" }, { "cell": "kimi-openrouter", "directional_speed_waves": [ true, false, true ], "opponent": "hermes", "opponent_successes": 3, "pft_all_in_output_estimate_advantage_x_total": 1.4337, "pft_all_in_output_estimate_savings_percent": 30.25, "pft_cost_advantage_x_total": 1.4427, "pft_cost_savings_percent": 30.68, "pft_speedup_x_total": 1.2665, "pft_successes": 3, "speed_result": "mixed_or_incomplete", "workload": "visual_site" }, { "cell": "opus", "directional_speed_waves": [ true, true ], "opponent": "cc", "opponent_successes": 2, "pft_all_in_output_estimate_advantage_x_total": 2.1009, "pft_all_in_output_estimate_savings_percent": 52.4, "pft_cost_advantage_x_total": 2.318, "pft_cost_savings_percent": 56.86, "pft_speedup_x_total": 1.7433, "pft_successes": 3, "speed_result": "mixed_or_incomplete", "workload": "visual_site" } ], "conformance_exclusions": [ { "artifact": "visual/results/glm-openrouter/hermes/wave2/agent_run.json", "cell": "glm-openrouter", "lane": "hermes", "reasons": [ "Hermes terminal isolation removed the supplied OPENAI_API_KEY.", "The agent generated final assets through the ChatGPT Codex OAuth backend using gpt-image-2-codex rather than the required OpenAI Image API gpt-image-2 route." ], "wave": 2, "workload": "visual_site" }, { "artifact": "visual/results/glm-openrouter/hermes/wave3/agent_run.json", "cell": "glm-openrouter", "lane": "hermes", "reasons": [ "Hermes terminal isolation removed the supplied OPENAI_API_KEY.", "The agent generated final assets through a discovered Vercel AI Gateway credential rather than the matched direct OpenAI Image API route." ], "wave": 3, "workload": "visual_site" } ], "judge_overhead": { "cache_write_tokens": 618624, "cached_input_tokens": 0, "estimated_cost_usd": 6.11742, "input_tokens": 618678, "output_tokens": 75025, "passes": 18, "source": "GPT-5.6-Sol response usage reconstructed with official standard short-context prices" }, "raw_runs": [ { "artifact": "visual/results/glm-openrouter/hermes/wave1/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 1.931643, "contestant_plus_image_output_estimate_usd": 1.720643, "cost_usd": 1.179643, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.752, "image_output_estimate_usd": 0.541, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 1, "wall_seconds": 1288.428, "wave": 1, "workload": "visual_site" }, { "artifact": "visual/results/glm-openrouter/hermes/wave2/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [ "Hermes terminal isolation removed the supplied OPENAI_API_KEY.", "The agent generated final assets through the ChatGPT Codex OAuth backend using gpt-image-2-codex rather than the required OpenAI Image API gpt-image-2 route." ], "conformance_valid": false, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 1.003355, "final_image_count": 3, "image_cost_evidence": "attempt_audit_unsupported_route_or_price", "image_count": 5, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": false, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1456.798, "wave": 2, "workload": "visual_site" }, { "artifact": "visual/results/glm-openrouter/hermes/wave3/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [ "Hermes terminal isolation removed the supplied OPENAI_API_KEY.", "The agent generated final assets through a discovered Vercel AI Gateway credential rather than the matched direct OpenAI Image API route." ], "conformance_valid": false, "contestant_plus_image_output_estimate_upper_bound_usd": 3.072207, "contestant_plus_image_output_estimate_usd": 2.990207, "cost_usd": 1.558207, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 15, "image_output_estimate_upper_bound_usd": 1.514, "image_output_estimate_usd": 1.432, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": false, "timed_out": false, "uncertain_image_count": 2, "wall_seconds": 2621.121, "wave": 3, "workload": "visual_site" }, { "artifact": "visual/results/glm-openrouter/pft/wave1/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 1.706904, "contestant_plus_image_output_estimate_usd": 1.706904, "cost_usd": 0.328904, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 10, "image_output_estimate_upper_bound_usd": 1.378, "image_output_estimate_usd": 1.378, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 36, "finish_reasons": { "ok": 36 }, "providers": { "StreamLake": 36 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1836.331, "wave": 1, "workload": "visual_site" }, { "artifact": "visual/results/glm-openrouter/pft/wave2/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 1.561625, "contestant_plus_image_output_estimate_usd": 1.561625, "cost_usd": 0.855625, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 4, "image_output_estimate_upper_bound_usd": 0.706, "image_output_estimate_usd": 0.706, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 67, "finish_reasons": { "ok": 67 }, "providers": { "Alibaba": 37, "Novita": 8, "StreamLake": 22 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1497.89, "wave": 2, "workload": "visual_site" }, { "artifact": "visual/results/glm-openrouter/pft/wave3/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 0.961547, "contestant_plus_image_output_estimate_usd": 0.961547, "cost_usd": 0.374547, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.587, "image_output_estimate_usd": 0.587, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 40, "finish_reasons": { "ok": 40 }, "providers": { "StreamLake": 40 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1220.0, "wave": 3, "workload": "visual_site" }, { "artifact": "visual/results/kimi-openrouter/hermes/wave1/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 4.455852, "contestant_plus_image_output_estimate_usd": 4.455852, "cost_usd": 3.465852, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 6, "image_output_estimate_upper_bound_usd": 0.99, "image_output_estimate_usd": 0.99, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1840.704, "wave": 1, "workload": "visual_site" }, { "artifact": "visual/results/kimi-openrouter/hermes/wave2/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 2.667302, "contestant_plus_image_output_estimate_usd": 2.667302, "cost_usd": 2.172302, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.495, "image_output_estimate_usd": 0.495, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1388.9, "wave": 2, "workload": "visual_site" }, { "artifact": "visual/results/kimi-openrouter/hermes/wave3/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 2.42681, "contestant_plus_image_output_estimate_usd": 2.42681, "cost_usd": 1.93181, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.495, "image_output_estimate_usd": 0.495, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 2393.333, "wave": 3, "workload": "visual_site" }, { "artifact": "visual/results/kimi-openrouter/pft/wave1/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 2.181337, "contestant_plus_image_output_estimate_usd": 2.181337, "cost_usd": 1.686337, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.495, "image_output_estimate_usd": 0.495, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 42, "finish_reasons": { "ok": 42 }, "providers": { "Fireworks": 16, "Moonshot AI": 26 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1292.03, "wave": 1, "workload": "visual_site" }, { "artifact": "visual/results/kimi-openrouter/pft/wave2/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 1.88888, "contestant_plus_image_output_estimate_usd": 1.88888, "cost_usd": 1.46488, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 4, "image_output_estimate_upper_bound_usd": 0.424, "image_output_estimate_usd": 0.424, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 40, "finish_reasons": { "ok": 40 }, "providers": { "Moonshot AI": 40 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1546.911, "wave": 2, "workload": "visual_site" }, { "artifact": "visual/results/kimi-openrouter/pft/wave3/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 2.590994, "contestant_plus_image_output_estimate_usd": 2.590994, "cost_usd": 2.095994, "final_image_count": 3, "image_cost_evidence": "attempt_audit", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.495, "image_output_estimate_usd": 0.495, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 66, "finish_reasons": { "ok": 65, "stream closed before chat completions finished": 1 }, "providers": { "Fireworks": 23, "Moonshot AI": 43 }, "retry_attempts": 1 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1600.704, "wave": 3, "workload": "visual_site" }, { "artifact": "visual/results/opus/cc/wave1/agent_run.json", "cell": "opus", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 5.627564, "contestant_plus_image_output_estimate_usd": 5.627564, "cost_usd": 5.08656375, "final_image_count": 3, "image_cost_evidence": "final_manifest_only", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.541, "image_output_estimate_usd": 0.541, "lane": "cc", "model": "claude-opus-5", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1420.029, "wave": 1, "workload": "visual_site" }, { "artifact": "visual/results/opus/cc/wave2/agent_run.json", "cell": "opus", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 7.183806, "contestant_plus_image_output_estimate_usd": 7.183806, "cost_usd": 6.6428055, "final_image_count": 3, "image_cost_evidence": "final_manifest_only", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.541, "image_output_estimate_usd": 0.541, "lane": "cc", "model": "claude-opus-5", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1689.07, "wave": 2, "workload": "visual_site" }, { "artifact": "visual/results/opus/cc/wave3/agent_run.json", "cell": "opus", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 8.921946, "contestant_plus_image_output_estimate_usd": 8.921946, "cost_usd": 8.380946, "final_image_count": 3, "image_cost_evidence": "final_manifest_only", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.541, "image_output_estimate_usd": 0.541, "lane": "cc", "model": "claude-opus-5", "pft_call_diagnostics": null, "returncode": 1, "success": false, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 2228.567, "wave": 3, "workload": "visual_site" }, { "artifact": "visual/results/opus/pft/wave1/agent_run.json", "cell": "opus", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 3.39652, "contestant_plus_image_output_estimate_usd": 3.39652, "cost_usd": 2.8555205, "final_image_count": 3, "image_cost_evidence": "final_manifest_only", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.541, "image_output_estimate_usd": 0.541, "lane": "pft", "model": "claude-opus-5", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 954.982, "wave": 1, "workload": "visual_site" }, { "artifact": "visual/results/opus/pft/wave2/agent_run.json", "cell": "opus", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 3.588637, "contestant_plus_image_output_estimate_usd": 3.588637, "cost_usd": 3.00163725, "final_image_count": 3, "image_cost_evidence": "final_manifest_only", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.587, "image_output_estimate_usd": 0.587, "lane": "pft", "model": "claude-opus-5", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 1118.881, "wave": 2, "workload": "visual_site" }, { "artifact": "visual/results/opus/pft/wave3/agent_run.json", "cell": "opus", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": 3.359437, "contestant_plus_image_output_estimate_usd": 3.359437, "cost_usd": 2.81843725, "final_image_count": 3, "image_cost_evidence": "final_manifest_only", "image_count": 3, "image_output_estimate_upper_bound_usd": 0.541, "image_output_estimate_usd": 0.541, "lane": "pft", "model": "claude-opus-5", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 988.026, "wave": 3, "workload": "visual_site" }, { "artifact": "deterministic/eventforge/glm-openrouter/wave1/openrouter/results/hermes/wave1/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.104977, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 463.925, "wave": 1, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/glm-openrouter/wave1/openrouter/results/pft/wave1/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.050411, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 6, "finish_reasons": { "ok": 6 }, "providers": { "Sail Research": 2, "Together": 4 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 259.933, "wave": 1, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/glm-openrouter/wave2/openrouter/results/hermes/wave2/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.150947, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 750.359, "wave": 2, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/glm-openrouter/wave2/openrouter/results/pft/wave2/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.078456, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 8, "finish_reasons": { "ok": 8 }, "providers": { "Friendli": 8 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 654.724, "wave": 2, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/glm-openrouter/wave3/openrouter/results/hermes/wave3/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.183729, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 516.232, "wave": 3, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/glm-openrouter/wave3/openrouter/results/pft/wave3/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.021822, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 7, "finish_reasons": { "ok": 7 }, "providers": { "AtlasCloud": 3, "Decart": 1, "Fireworks": 1, "Inceptron": 2 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 265.688, "wave": 3, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/kimi-openrouter/wave1/openrouter/results/hermes/wave1/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.356091, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 277.521, "wave": 1, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/kimi-openrouter/wave1/openrouter/results/pft/wave1/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.731491, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 16, "finish_reasons": { "ok": 16 }, "providers": { "Fireworks": 10, "Moonshot AI": 6 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 710.7, "wave": 1, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/kimi-openrouter/wave2/openrouter/results/hermes/wave2/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.986039, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": -15, "success": false, "timed_out": true, "uncertain_image_count": 0, "wall_seconds": 1200.032, "wave": 2, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/kimi-openrouter/wave2/openrouter/results/pft/wave2/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.348651, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 9, "finish_reasons": { "ok": 9 }, "providers": { "Fireworks": 1, "Moonshot AI": 8 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 582.386, "wave": 2, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/kimi-openrouter/wave3/openrouter/results/hermes/wave3/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.263867, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 314.765, "wave": 3, "workload": "eventforge" }, { "artifact": "deterministic/eventforge/kimi-openrouter/wave3/openrouter/results/pft/wave3/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.060133, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 6, "finish_reasons": { "ok": 6 }, "providers": { "BaseTen": 1, "Fireworks": 5 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 278.558, "wave": 3, "workload": "eventforge" }, { "artifact": "deterministic/queuecraft/glm-openrouter/wave1/openrouter/results/hermes/wave1/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.323866, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 513.471, "wave": 1, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-openrouter/wave1/openrouter/results/pft/wave1/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.165116, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 19, "finish_reasons": { "ok": 19 }, "providers": { "AtlasCloud": 19 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 114.527, "wave": 1, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-openrouter/wave2/openrouter/results/hermes/wave2/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.200992, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 288.692, "wave": 2, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-openrouter/wave2/openrouter/results/pft/wave2/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.527677, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 24, "finish_reasons": { "ok": 24 }, "providers": { "Alibaba": 7, "CoreWeave": 16, "Ionstream": 1 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 377.436, "wave": 2, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-openrouter/wave3/openrouter/results/hermes/wave3/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.498547, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "z-ai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 261.392, "wave": 3, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-openrouter/wave3/openrouter/results/pft/wave3/agent_run.json", "cell": "glm-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.272193, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "z-ai/glm-5.2", "pft_call_diagnostics": { "calls": 16, "finish_reasons": { "ok": 16 }, "providers": { "CoreWeave": 10, "SiliconFlow": 6 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 465.68, "wave": 3, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-vercel-current-hermes/wave1/vercel/results/hermes/wave1/agent_run.json", "cell": "glm-vercel", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.600463, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "zai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 307.413, "wave": 1, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-vercel-current-hermes/wave1/vercel/results/pft/wave1/agent_run.json", "cell": "glm-vercel", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.359863, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "zai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 134.213, "wave": 1, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-vercel-current-hermes/wave2/vercel/results/hermes/wave2/agent_run.json", "cell": "glm-vercel", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.566592, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "zai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 304.641, "wave": 2, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-vercel-current-hermes/wave2/vercel/results/pft/wave2/agent_run.json", "cell": "glm-vercel", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.392805, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "zai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 183.981, "wave": 2, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-vercel-current-hermes/wave3/vercel/results/hermes/wave3/agent_run.json", "cell": "glm-vercel", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.572873, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "zai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 178.317, "wave": 3, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/glm-vercel-current-hermes/wave3/vercel/results/pft/wave3/agent_run.json", "cell": "glm-vercel", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.295302, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "zai/glm-5.2", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 158.72, "wave": 3, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/kimi-openrouter/wave1/openrouter/results/hermes/wave1/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 2.062446, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 705.385, "wave": 1, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/kimi-openrouter/wave1/openrouter/results/pft/wave1/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.202949, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 9, "finish_reasons": { "ok": 9 }, "providers": { "Moonshot AI": 9 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 143.329, "wave": 1, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/kimi-openrouter/wave2/openrouter/results/hermes/wave2/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 1.191236, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 271.082, "wave": 2, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/kimi-openrouter/wave2/openrouter/results/pft/wave2/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.34042, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 11, "finish_reasons": { "ok": 11 }, "providers": { "Fireworks": 5, "Moonshot AI": 6 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 119.531, "wave": 2, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/kimi-openrouter/wave3/openrouter/results/hermes/wave3/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 1.549447, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "hermes", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": null, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 868.005, "wave": 3, "workload": "queuecraft" }, { "artifact": "deterministic/queuecraft/kimi-openrouter/wave3/openrouter/results/pft/wave3/agent_run.json", "cell": "kimi-openrouter", "conformance_reasons": [], "conformance_valid": true, "contestant_plus_image_output_estimate_upper_bound_usd": null, "contestant_plus_image_output_estimate_usd": null, "cost_usd": 0.371941, "final_image_count": 0, "image_cost_evidence": null, "image_count": 0, "image_output_estimate_upper_bound_usd": null, "image_output_estimate_usd": null, "lane": "pft", "model": "moonshotai/kimi-k3", "pft_call_diagnostics": { "calls": 12, "finish_reasons": { "ok": 12 }, "providers": { "Fireworks": 6, "Moonshot AI": 6 }, "retry_attempts": 0 }, "returncode": 0, "success": true, "timed_out": false, "uncertain_image_count": 0, "wall_seconds": 172.926, "wave": 3, "workload": "queuecraft" } ], "visual_judgments": [ { "cell": "glm-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.98, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_097873a303841d2f016a68849e692081a1a29e7cf1ec1b069d", "site_a_total": 81, "site_b_total": 97, "underlying_winner": "pft", "usage": { "input_tokens": 31575, "input_tokens_details": { "cache_write_tokens": 31572, "cached_tokens": 0 }, "output_tokens": 4619, "output_tokens_details": { "reasoning_tokens": 3505 }, "total_tokens": 36194 } }, { "confidence": 0.97, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_099bf309fe88488e016a6884f8786c81a3a24b0edcd37c8f3f", "site_a_total": 95, "site_b_total": 80, "underlying_winner": "pft", "usage": { "input_tokens": 31575, "input_tokens_details": { "cache_write_tokens": 31572, "cached_tokens": 0 }, "output_tokens": 5510, "output_tokens_details": { "reasoning_tokens": 4248 }, "total_tokens": 37085 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 1 }, { "cell": "glm-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.95, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_02f4b76f1900fdae016a68856a5c14819dbc726b9f2afd45a9", "site_a_total": 94, "site_b_total": 85, "underlying_winner": "hermes", "usage": { "input_tokens": 34437, "input_tokens_details": { "cache_write_tokens": 34434, "cached_tokens": 0 }, "output_tokens": 4249, "output_tokens_details": { "reasoning_tokens": 3106 }, "total_tokens": 38686 } }, { "confidence": 0.9, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_04200e66a12eb497016a6885c466b0819e80e7e5c199ce0678", "site_a_total": 95, "site_b_total": 90, "underlying_winner": "pft", "usage": { "input_tokens": 34437, "input_tokens_details": { "cache_write_tokens": 34434, "cached_tokens": 0 }, "output_tokens": 3586, "output_tokens_details": { "reasoning_tokens": 2378 }, "total_tokens": 38023 } } ], "verdict": "tie_inconclusive_order_sensitive", "verdict_reason": "the balanced A/B orders did not select the same underlying site", "wave": 2 }, { "cell": "glm-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.97, "order": { "a": "pft", "b": "hermes" }, "pass": "normal", "response_id": "resp_06e7d8151aff76f5016a688607a2c881a28fa3c54ec89ee566", "site_a_total": 93, "site_b_total": 84, "underlying_winner": "pft", "usage": { "input_tokens": 33141, "input_tokens_details": { "cache_write_tokens": 33138, "cached_tokens": 0 }, "output_tokens": 3572, "output_tokens_details": { "reasoning_tokens": 2588 }, "total_tokens": 36713 } }, { "confidence": 0.93, "order": { "a": "hermes", "b": "pft" }, "pass": "swapped", "response_id": "resp_04f91257ce3a8bae016a68864ec60c819f8be4d52fb64e7db9", "site_a_total": 87, "site_b_total": 94, "underlying_winner": "pft", "usage": { "input_tokens": 33141, "input_tokens_details": { "cache_write_tokens": 33138, "cached_tokens": 0 }, "output_tokens": 4768, "output_tokens_details": { "reasoning_tokens": 3587 }, "total_tokens": 37909 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 3 }, { "cell": "kimi-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.97, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_0c2c99f3ba27bc1c016a6882d48dbc81a288f46c257b110001", "site_a_total": 90, "site_b_total": 98, "underlying_winner": "pft", "usage": { "input_tokens": 29793, "input_tokens_details": { "cache_write_tokens": 29790, "cached_tokens": 0 }, "output_tokens": 3127, "output_tokens_details": { "reasoning_tokens": 2070 }, "total_tokens": 32920 } }, { "confidence": 0.95, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_058c66903f74b8f4016a688315b67c8191b610073acbeca3b8", "site_a_total": 97, "site_b_total": 90, "underlying_winner": "pft", "usage": { "input_tokens": 29793, "input_tokens_details": { "cache_write_tokens": 29790, "cached_tokens": 0 }, "output_tokens": 3500, "output_tokens_details": { "reasoning_tokens": 2554 }, "total_tokens": 33293 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 1 }, { "cell": "kimi-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.88, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_09bee529677f6e61016a68835a81a48191aa8a4ce383ca67cf", "site_a_total": 92, "site_b_total": 96, "underlying_winner": "pft", "usage": { "input_tokens": 31467, "input_tokens_details": { "cache_write_tokens": 31464, "cached_tokens": 0 }, "output_tokens": 3654, "output_tokens_details": { "reasoning_tokens": 2342 }, "total_tokens": 35121 } }, { "confidence": 0.84, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_00e7c2a5b07dcf15016a6883a03ce881a2a41ce13291f428bb", "site_a_total": 96, "site_b_total": 92, "underlying_winner": "pft", "usage": { "input_tokens": 31467, "input_tokens_details": { "cache_write_tokens": 31464, "cached_tokens": 0 }, "output_tokens": 4139, "output_tokens_details": { "reasoning_tokens": 2978 }, "total_tokens": 35606 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 2 }, { "cell": "kimi-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.92, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_044eaa94865f9638016a6883efd628819c858f0184f826199c", "site_a_total": 90, "site_b_total": 94, "underlying_winner": "pft", "usage": { "input_tokens": 31251, "input_tokens_details": { "cache_write_tokens": 31248, "cached_tokens": 0 }, "output_tokens": 4810, "output_tokens_details": { "reasoning_tokens": 3506 }, "total_tokens": 36061 } }, { "confidence": 0.9, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_09b50af7a87e7226016a688453036c81a09e7a41f9161c9aae", "site_a_total": 94, "site_b_total": 88, "underlying_winner": "pft", "usage": { "input_tokens": 31251, "input_tokens_details": { "cache_write_tokens": 31248, "cached_tokens": 0 }, "output_tokens": 3739, "output_tokens_details": { "reasoning_tokens": 2433 }, "total_tokens": 34990 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 3 }, { "cell": "opus", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.95, "order": { "a": "cc", "b": "pft" }, "pass": "normal", "response_id": "resp_04a2bf03abdcc12c016a681d41e7ac819d93e2fadebc253660", "site_a_total": 94, "site_b_total": 90, "underlying_winner": "cc", "usage": { "input_tokens": 38703, "input_tokens_details": { "cache_write_tokens": 38700, "cached_tokens": 0 }, "output_tokens": 4085, "output_tokens_details": { "reasoning_tokens": 2921 }, "total_tokens": 42788 } }, { "confidence": 0.94, "order": { "a": "pft", "b": "cc" }, "pass": "swapped", "response_id": "resp_0e996b0759b36842016a681d86fb60819f9fd673a6adb6cecb", "site_a_total": 91, "site_b_total": 97, "underlying_winner": "cc", "usage": { "input_tokens": 38703, "input_tokens_details": { "cache_write_tokens": 38700, "cached_tokens": 0 }, "output_tokens": 4167, "output_tokens_details": { "reasoning_tokens": 2995 }, "total_tokens": 42870 } } ], "verdict": "cc", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 1 }, { "cell": "opus", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.94, "order": { "a": "cc", "b": "pft" }, "pass": "normal", "response_id": "resp_04949cbcc4b7c72b016a681dd1b4bc81a3957c183d81374832", "site_a_total": 89, "site_b_total": 95, "underlying_winner": "pft", "usage": { "input_tokens": 38001, "input_tokens_details": { "cache_write_tokens": 37998, "cached_tokens": 0 }, "output_tokens": 4050, "output_tokens_details": { "reasoning_tokens": 3106 }, "total_tokens": 42051 } }, { "confidence": 0.96, "order": { "a": "pft", "b": "cc" }, "pass": "swapped", "response_id": "resp_09ead2c455dafb03016a681e1d7dd0819d8e4316196190292d", "site_a_total": 95, "site_b_total": 88, "underlying_winner": "pft", "usage": { "input_tokens": 38001, "input_tokens_details": { "cache_write_tokens": 37998, "cached_tokens": 0 }, "output_tokens": 3225, "output_tokens_details": { "reasoning_tokens": 1999 }, "total_tokens": 41226 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 2 }, { "cell": "opus", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.96, "order": { "a": "cc", "b": "pft" }, "pass": "normal", "response_id": "resp_08d806ead87241bc016a681e5c0cfc81a285668614d0e28ad3", "site_a_total": 97, "site_b_total": 97, "underlying_winner": "tie", "usage": { "input_tokens": 40971, "input_tokens_details": { "cache_write_tokens": 40968, "cached_tokens": 0 }, "output_tokens": 5534, "output_tokens_details": { "reasoning_tokens": 4469 }, "total_tokens": 46505 } }, { "confidence": 0.91, "order": { "a": "pft", "b": "cc" }, "pass": "swapped", "response_id": "resp_0c64dac8f47e4d07016a681ebed364819296de2b4cbe22b939", "site_a_total": 96, "site_b_total": 93, "underlying_winner": "pft", "usage": { "input_tokens": 40971, "input_tokens_details": { "cache_write_tokens": 40968, "cached_tokens": 0 }, "output_tokens": 4691, "output_tokens_details": { "reasoning_tokens": 3403 }, "total_tokens": 45662 } } ], "verdict": "tie_inconclusive_order_sensitive", "verdict_reason": "the balanced A/B orders did not select the same underlying site", "wave": 3 } ], "visual_verdict_summary": [ { "cell": "glm-openrouter", "inconclusive": 1, "opponent": "hermes", "opponent_wins": 0, "pft_wins": 2, "waves": 3 }, { "cell": "kimi-openrouter", "inconclusive": 0, "opponent": "hermes", "opponent_wins": 0, "pft_wins": 3, "waves": 3 }, { "cell": "opus", "inconclusive": 1, "opponent": "cc", "opponent_wins": 1, "pft_wins": 1, "waves": 3 } ] }