196 lines
6.1 KiB
JSON
196 lines
6.1 KiB
JSON
{
|
|
"measurement": "live_working_brief_trial_comparison_v1",
|
|
"costs": {
|
|
"previous_eur": 0.34,
|
|
"current_eur": 0.31,
|
|
"current_balance_start_eur": 7.4,
|
|
"current_balance_end_eur": 7.71
|
|
},
|
|
"test_data_bytes_after_run": 18092447774,
|
|
"comparisons": [
|
|
{
|
|
"workspace": "service-data\\trials\\bug17-working-brief-20261016",
|
|
"generation": 114,
|
|
"calls": 21,
|
|
"investigation_calls": 17,
|
|
"tokens": {
|
|
"input_tokens": 576864,
|
|
"output_tokens": 45814,
|
|
"reasoning_tokens": 31279,
|
|
"cached_input_tokens": 367761,
|
|
"cache_created_tokens": 43266,
|
|
"elapsed_seconds": 409.32800000000003,
|
|
"estimated_cache_storage_cost": 0.0018027499999999999
|
|
},
|
|
"inv_tokens": {
|
|
"input_tokens": 506697,
|
|
"output_tokens": 38473,
|
|
"reasoning_tokens": 26767,
|
|
"cached_input_tokens": 367761,
|
|
"elapsed_seconds": 355.26500000000004
|
|
},
|
|
"cache_statuses": {
|
|
"created": 2,
|
|
"reused": 15
|
|
},
|
|
"cached_tokens_per_inv": {
|
|
"21633": 17
|
|
},
|
|
"live_total": 396109,
|
|
"errors": [
|
|
{
|
|
"hypothesis": "H1",
|
|
"code": "INVESTIGATION_BRIEF_CHANGE",
|
|
"kind": "invalid_response",
|
|
"llm_ref": "llm/operations/b5de43dfa1122fa86decfa538047ee813621839eded8ed15356d341f3439e383/round-0001.json"
|
|
},
|
|
{
|
|
"hypothesis": "H2",
|
|
"code": "LLM_OUTPUT_SCHEMA",
|
|
"kind": "invalid_response",
|
|
"llm_ref": "llm/operations/a4ed5334e4125ed14bcbca18629a561513bba48ef92afecc570c14d9e90e05cc/round-0001.json"
|
|
},
|
|
{
|
|
"hypothesis": "H2",
|
|
"code": "INVESTIGATION_BRIEF_REFERENCE",
|
|
"kind": "invalid_response",
|
|
"llm_ref": "llm/operations/ddf237df888a860f57b91498528fd599614514dff235353c4f61410ff3594423/round-0001.json"
|
|
},
|
|
{
|
|
"hypothesis": "H2",
|
|
"code": "INVESTIGATION_BRIEF_CHANGE",
|
|
"kind": "invalid_response",
|
|
"llm_ref": "llm/operations/23e886f4a2c5eeadfea37a9114a2282780ab8a12c14b970fdcc26da184fc793b/round-0001.json"
|
|
},
|
|
{
|
|
"hypothesis": "H2",
|
|
"code": "INVESTIGATION_CAUSAL_EFFECT",
|
|
"kind": "invalid_response",
|
|
"llm_ref": "llm/operations/dc6beafc7db15bec5af22b3b34f0fd40506f577c3921f56d87190df9faf8d348/round-0001.json"
|
|
},
|
|
{
|
|
"hypothesis": "H2",
|
|
"code": "INVESTIGATION_BRIEF_REFERENCE",
|
|
"kind": "invalid_response",
|
|
"llm_ref": "llm/operations/bbf03137db61614a7898353127aae3d114e56606ca757881038446b590a4cc21/round-0001.json"
|
|
},
|
|
{
|
|
"hypothesis": "H3",
|
|
"code": "LLM_OUTPUT_SCHEMA",
|
|
"kind": "invalid_response",
|
|
"llm_ref": "llm/operations/71fd77c20b2af9bb2d842927c7a39cc8e1cf3e15541f387a26054fdb1047b6e4/round-0001.json"
|
|
}
|
|
],
|
|
"hypotheses": {
|
|
"H1": {
|
|
"confidence": "weakened",
|
|
"disposition": "active",
|
|
"iterations": 5,
|
|
"invalid_responses": 0
|
|
},
|
|
"H2": {
|
|
"confidence": "plausible",
|
|
"disposition": "active",
|
|
"iterations": 5,
|
|
"invalid_responses": 1
|
|
},
|
|
"H3": {
|
|
"confidence": "unassessed",
|
|
"disposition": "active",
|
|
"iterations": 1,
|
|
"invalid_responses": 0
|
|
}
|
|
},
|
|
"duration_seconds": 522.340196,
|
|
"rejected_calls_usage": {
|
|
"input_tokens": 219430,
|
|
"output_tokens": 16370,
|
|
"reasoning_tokens": 11234,
|
|
"cached_input_tokens": 151431,
|
|
"elapsed_seconds": 139.73299999999998,
|
|
"estimated_cost": 0.123744075
|
|
}
|
|
},
|
|
{
|
|
"workspace": "service-data\\trials\\bug17-stable-case-cache-20261016",
|
|
"generation": 148,
|
|
"calls": 21,
|
|
"investigation_calls": 17,
|
|
"tokens": {
|
|
"input_tokens": 658995,
|
|
"output_tokens": 31378,
|
|
"reasoning_tokens": 22227,
|
|
"cached_input_tokens": 339779,
|
|
"cache_created_tokens": 79948,
|
|
"elapsed_seconds": 275.453,
|
|
"estimated_cache_storage_cost": 0.0033311666666666667
|
|
},
|
|
"inv_tokens": {
|
|
"input_tokens": 589053,
|
|
"output_tokens": 24345,
|
|
"reasoning_tokens": 18356,
|
|
"cached_input_tokens": 339779,
|
|
"elapsed_seconds": 219.109
|
|
},
|
|
"cache_statuses": {
|
|
"created": 4,
|
|
"reused": 13
|
|
},
|
|
"cached_tokens_per_inv": {
|
|
"19987": 17
|
|
},
|
|
"live_total": 645032,
|
|
"errors": [
|
|
{
|
|
"hypothesis": "H1",
|
|
"code": "PROJECT_SOURCE_RANGE",
|
|
"kind": "invalid_response",
|
|
"llm_ref": "llm/operations/9ca4b010c7ebc9e0ae40a106866bbb196dd23366b2e453b50c4a6e629b9f6874/round-0001.json"
|
|
}
|
|
],
|
|
"hypotheses": {
|
|
"H1": {
|
|
"confidence": "supported",
|
|
"disposition": "active",
|
|
"iterations": 13,
|
|
"invalid_responses": 0
|
|
},
|
|
"H2": {
|
|
"confidence": "unassessed",
|
|
"disposition": "active",
|
|
"iterations": 1,
|
|
"invalid_responses": 0
|
|
},
|
|
"H3": {
|
|
"confidence": "unassessed",
|
|
"disposition": "active",
|
|
"iterations": 1,
|
|
"invalid_responses": 0
|
|
},
|
|
"H4": {
|
|
"confidence": "unassessed",
|
|
"disposition": "active",
|
|
"iterations": 1,
|
|
"invalid_responses": 0
|
|
}
|
|
},
|
|
"duration_seconds": 1202.567749,
|
|
"rejected_calls_usage": {
|
|
"input_tokens": 43739,
|
|
"output_tokens": 1166,
|
|
"reasoning_tokens": 820,
|
|
"cached_input_tokens": 19987,
|
|
"elapsed_seconds": 11.0,
|
|
"estimated_cost": 0.023685525
|
|
}
|
|
}
|
|
],
|
|
"limitations": [
|
|
"Both runs stopped at 20 dispatches, neither completed.",
|
|
"Different hypothesis sets and 7 versus 1 rejected replies; not a controlled equal-progress cost benchmark.",
|
|
"Input tokens include cached tokens. Output includes reasoning; reasoning is not additional to output.",
|
|
"Provider elapsed times are not total runtime.",
|
|
"No additional paid API calls during evaluation."
|
|
]
|
|
}
|