[ { "cell": "glm-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.98, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_097873a303841d2f016a68849e692081a1a29e7cf1ec1b069d", "site_a_total": 81, "site_b_total": 97, "underlying_winner": "pft", "usage": { "input_tokens": 31575, "input_tokens_details": { "cache_write_tokens": 31572, "cached_tokens": 0 }, "output_tokens": 4619, "output_tokens_details": { "reasoning_tokens": 3505 }, "total_tokens": 36194 } }, { "confidence": 0.97, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_099bf309fe88488e016a6884f8786c81a3a24b0edcd37c8f3f", "site_a_total": 95, "site_b_total": 80, "underlying_winner": "pft", "usage": { "input_tokens": 31575, "input_tokens_details": { "cache_write_tokens": 31572, "cached_tokens": 0 }, "output_tokens": 5510, "output_tokens_details": { "reasoning_tokens": 4248 }, "total_tokens": 37085 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 1 }, { "cell": "glm-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.95, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_02f4b76f1900fdae016a68856a5c14819dbc726b9f2afd45a9", "site_a_total": 94, "site_b_total": 85, "underlying_winner": "hermes", "usage": { "input_tokens": 34437, "input_tokens_details": { "cache_write_tokens": 34434, "cached_tokens": 0 }, "output_tokens": 4249, "output_tokens_details": { "reasoning_tokens": 3106 }, "total_tokens": 38686 } }, { "confidence": 0.9, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_04200e66a12eb497016a6885c466b0819e80e7e5c199ce0678", "site_a_total": 95, "site_b_total": 90, "underlying_winner": "pft", "usage": { "input_tokens": 34437, "input_tokens_details": { "cache_write_tokens": 34434, "cached_tokens": 0 }, "output_tokens": 3586, "output_tokens_details": { "reasoning_tokens": 2378 }, "total_tokens": 38023 } } ], "verdict": "tie_inconclusive_order_sensitive", "verdict_reason": "the balanced A/B orders did not select the same underlying site", "wave": 2 }, { "cell": "glm-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.97, "order": { "a": "pft", "b": "hermes" }, "pass": "normal", "response_id": "resp_06e7d8151aff76f5016a688607a2c881a28fa3c54ec89ee566", "site_a_total": 93, "site_b_total": 84, "underlying_winner": "pft", "usage": { "input_tokens": 33141, "input_tokens_details": { "cache_write_tokens": 33138, "cached_tokens": 0 }, "output_tokens": 3572, "output_tokens_details": { "reasoning_tokens": 2588 }, "total_tokens": 36713 } }, { "confidence": 0.93, "order": { "a": "hermes", "b": "pft" }, "pass": "swapped", "response_id": "resp_04f91257ce3a8bae016a68864ec60c819f8be4d52fb64e7db9", "site_a_total": 87, "site_b_total": 94, "underlying_winner": "pft", "usage": { "input_tokens": 33141, "input_tokens_details": { "cache_write_tokens": 33138, "cached_tokens": 0 }, "output_tokens": 4768, "output_tokens_details": { "reasoning_tokens": 3587 }, "total_tokens": 37909 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 3 }, { "cell": "kimi-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.97, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_0c2c99f3ba27bc1c016a6882d48dbc81a288f46c257b110001", "site_a_total": 90, "site_b_total": 98, "underlying_winner": "pft", "usage": { "input_tokens": 29793, "input_tokens_details": { "cache_write_tokens": 29790, "cached_tokens": 0 }, "output_tokens": 3127, "output_tokens_details": { "reasoning_tokens": 2070 }, "total_tokens": 32920 } }, { "confidence": 0.95, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_058c66903f74b8f4016a688315b67c8191b610073acbeca3b8", "site_a_total": 97, "site_b_total": 90, "underlying_winner": "pft", "usage": { "input_tokens": 29793, "input_tokens_details": { "cache_write_tokens": 29790, "cached_tokens": 0 }, "output_tokens": 3500, "output_tokens_details": { "reasoning_tokens": 2554 }, "total_tokens": 33293 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 1 }, { "cell": "kimi-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.88, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_09bee529677f6e61016a68835a81a48191aa8a4ce383ca67cf", "site_a_total": 92, "site_b_total": 96, "underlying_winner": "pft", "usage": { "input_tokens": 31467, "input_tokens_details": { "cache_write_tokens": 31464, "cached_tokens": 0 }, "output_tokens": 3654, "output_tokens_details": { "reasoning_tokens": 2342 }, "total_tokens": 35121 } }, { "confidence": 0.84, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_00e7c2a5b07dcf15016a6883a03ce881a2a41ce13291f428bb", "site_a_total": 96, "site_b_total": 92, "underlying_winner": "pft", "usage": { "input_tokens": 31467, "input_tokens_details": { "cache_write_tokens": 31464, "cached_tokens": 0 }, "output_tokens": 4139, "output_tokens_details": { "reasoning_tokens": 2978 }, "total_tokens": 35606 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 2 }, { "cell": "kimi-openrouter", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.92, "order": { "a": "hermes", "b": "pft" }, "pass": "normal", "response_id": "resp_044eaa94865f9638016a6883efd628819c858f0184f826199c", "site_a_total": 90, "site_b_total": 94, "underlying_winner": "pft", "usage": { "input_tokens": 31251, "input_tokens_details": { "cache_write_tokens": 31248, "cached_tokens": 0 }, "output_tokens": 4810, "output_tokens_details": { "reasoning_tokens": 3506 }, "total_tokens": 36061 } }, { "confidence": 0.9, "order": { "a": "pft", "b": "hermes" }, "pass": "swapped", "response_id": "resp_09b50af7a87e7226016a688453036c81a09e7a41f9161c9aae", "site_a_total": 94, "site_b_total": 88, "underlying_winner": "pft", "usage": { "input_tokens": 31251, "input_tokens_details": { "cache_write_tokens": 31248, "cached_tokens": 0 }, "output_tokens": 3739, "output_tokens_details": { "reasoning_tokens": 2433 }, "total_tokens": 34990 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 3 }, { "cell": "opus", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.95, "order": { "a": "cc", "b": "pft" }, "pass": "normal", "response_id": "resp_04a2bf03abdcc12c016a681d41e7ac819d93e2fadebc253660", "site_a_total": 94, "site_b_total": 90, "underlying_winner": "cc", "usage": { "input_tokens": 38703, "input_tokens_details": { "cache_write_tokens": 38700, "cached_tokens": 0 }, "output_tokens": 4085, "output_tokens_details": { "reasoning_tokens": 2921 }, "total_tokens": 42788 } }, { "confidence": 0.94, "order": { "a": "pft", "b": "cc" }, "pass": "swapped", "response_id": "resp_0e996b0759b36842016a681d86fb60819f9fd673a6adb6cecb", "site_a_total": 91, "site_b_total": 97, "underlying_winner": "cc", "usage": { "input_tokens": 38703, "input_tokens_details": { "cache_write_tokens": 38700, "cached_tokens": 0 }, "output_tokens": 4167, "output_tokens_details": { "reasoning_tokens": 2995 }, "total_tokens": 42870 } } ], "verdict": "cc", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 1 }, { "cell": "opus", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.94, "order": { "a": "cc", "b": "pft" }, "pass": "normal", "response_id": "resp_04949cbcc4b7c72b016a681dd1b4bc81a3957c183d81374832", "site_a_total": 89, "site_b_total": 95, "underlying_winner": "pft", "usage": { "input_tokens": 38001, "input_tokens_details": { "cache_write_tokens": 37998, "cached_tokens": 0 }, "output_tokens": 4050, "output_tokens_details": { "reasoning_tokens": 3106 }, "total_tokens": 42051 } }, { "confidence": 0.96, "order": { "a": "pft", "b": "cc" }, "pass": "swapped", "response_id": "resp_09ead2c455dafb03016a681e1d7dd0819d8e4316196190292d", "site_a_total": 95, "site_b_total": 88, "underlying_winner": "pft", "usage": { "input_tokens": 38001, "input_tokens_details": { "cache_write_tokens": 37998, "cached_tokens": 0 }, "output_tokens": 3225, "output_tokens_details": { "reasoning_tokens": 1999 }, "total_tokens": 41226 } } ], "verdict": "pft", "verdict_reason": "both balanced A/B orders selected the same underlying site", "wave": 2 }, { "cell": "opus", "model": "gpt-5.6-sol", "passes": [ { "confidence": 0.96, "order": { "a": "cc", "b": "pft" }, "pass": "normal", "response_id": "resp_08d806ead87241bc016a681e5c0cfc81a285668614d0e28ad3", "site_a_total": 97, "site_b_total": 97, "underlying_winner": "tie", "usage": { "input_tokens": 40971, "input_tokens_details": { "cache_write_tokens": 40968, "cached_tokens": 0 }, "output_tokens": 5534, "output_tokens_details": { "reasoning_tokens": 4469 }, "total_tokens": 46505 } }, { "confidence": 0.91, "order": { "a": "pft", "b": "cc" }, "pass": "swapped", "response_id": "resp_0c64dac8f47e4d07016a681ebed364819296de2b4cbe22b939", "site_a_total": 96, "site_b_total": 93, "underlying_winner": "pft", "usage": { "input_tokens": 40971, "input_tokens_details": { "cache_write_tokens": 40968, "cached_tokens": 0 }, "output_tokens": 4691, "output_tokens_details": { "reasoning_tokens": 3403 }, "total_tokens": 45662 } } ], "verdict": "tie_inconclusive_order_sensitive", "verdict_reason": "the balanced A/B orders did not select the same underlying site", "wave": 3 } ]