Nico/VianoBugAnalysis
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record startup batch trial metrics and successful scoped repair example

Browse Source
This commit is contained in:
Nico committed 2026-10-20 13:42:12 +02:00
1 parent b33208363c
commit 377644ffa0
7 files changed
+603
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/test-results/token-usage/bug20-startup-batch-20261020-g00000067.json
+496
View File
@@ -0,0 +1,496 @@
{
"format": "viano-token-usage-report",
"format_version": 1,
"recorded_date": "2026-10-20",
"run": "bug20-startup-batch-20261020",
"code_commit": "b332083",
"case_id": "BUG-000020",
"package_id": "trial-startup-batch-BUG-000020-20261020",
"package_generation": 67,
"analysis_revision": 1,
"state": "INVESTIGATING_HYPOTHESES",
"source_package": "service-data/trials/bug20-startup-batch-20261020/cases/897a914b67a02e6109a6b567e377e43c72b90ab216bfb9d2dc24485a5a55db8d/85f677c2e2c1e217563aa83ce86ad38ef239eb1143795161d0c706dbba3a8d21/g00000067.zip",
"manifest_sha256": "cf721c7c2b152f9decc0c51d7c1574f120a0dd95ee8c33305472bc3f9abf6d46",
"account_eur": {
"start": 12.55,
"final": null,
"settlement": "pending"
},
"startup": {
"order": [
"H1",
"H2",
"H3"
],
"done": true,
"review_rounds": 1,
"confidence_after_current_snapshot": {
"H1": "strongly_supported",
"H2": "weakened",
"H3": "weakened"
}
},
"notes": [
"output_tokens already includes reasoning; never add reasoning twice",
"Unknown values stay null and known sums carry missing counts",
"API elapsed sums are not wall-clock case duration",
"accepted_by refers to the immediate phase artifact; contributes_to includes retained source calls behind scoped repair",
"Planning excluding repair calls is not the same as accepted-only or all necessary planning work",
"This is a partial investigation snapshot, not a final diagnosis"
],
"totals_all_calls": {
"call_count": 10,
"input_tokens": {
"known_sum": 138418,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 25494,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 20456,
"missing_call_count": 2
},
"visible_output_tokens": {
"known_sum": 4401,
"missing_call_count": 2
},
"cached_input_tokens": {
"known_sum": 37767,
"missing_call_count": 4
},
"api_elapsed_seconds_sum": 188.033
},
"totals_by_group": {
"entity_resolution": {
"call_count": 2,
"input_tokens": {
"known_sum": 46745,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 637,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 0,
"missing_call_count": 2
},
"visible_output_tokens": {
"known_sum": 0,
"missing_call_count": 2
},
"cached_input_tokens": {
"known_sum": 0,
"missing_call_count": 2
},
"api_elapsed_seconds_sum": 8.984
},
"investigation_followup": {
"call_count": 2,
"input_tokens": {
"known_sum": 27792,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 1309,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 1004,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 305,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 12154,
"missing_call_count": 0
},
"api_elapsed_seconds_sum": 14.11
},
"planning": {
"call_count": 2,
"input_tokens": {
"known_sum": 21352,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 12087,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 9006,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 3081,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 0,
"missing_call_count": 2
},
"api_elapsed_seconds_sum": 78.922
},
"startup_joint_review": {
"call_count": 1,
"input_tokens": {
"known_sum": 9442,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 2953,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 2239,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 714,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 2060,
"missing_call_count": 0
},
"api_elapsed_seconds_sum": 22.204
},
"startup_queries": {
"call_count": 3,
"input_tokens": {
"known_sum": 33087,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 8508,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 8207,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 301,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 23553,
"missing_call_count": 0
},
"api_elapsed_seconds_sum": 63.813
}
},
"totals_planning_excluding_corrections": {
"call_count": 1,
"input_tokens": {
"known_sum": 13886,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 5878,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 3057,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 2821,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 0,
"missing_call_count": 1
},
"api_elapsed_seconds_sum": 39.672
},
"totals_planning_contributing_calls": {
"call_count": 2,
"input_tokens": {
"known_sum": 21352,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 12087,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 9006,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 3081,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 0,
"missing_call_count": 2
},
"api_elapsed_seconds_sum": 78.922
},
"calls": [
{
"call_ref": "llm/operations/6c0a36f93bea41b87241a30769bd92bb6dc387869b46d3a013ec595cfc7e9548/round-0001.json",
"call_sha256": "1a030b2a58045b475ba33e8652dd659432aeb9f522a375a41f38680a4202332e",
"phase": "INTERPRETING_CASE",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [],
"contributes_to": [],
"group": "entity_resolution",
"prompt_version": "entity_resolution_plan_v4",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 2.984,
"usage": {
"input_tokens": 21007,
"output_tokens": 166,
"reasoning_tokens": null,
"cached_input_tokens": null,
"uncached_input_tokens": null,
"visible_output_tokens": null
}
},
{
"call_ref": "llm/operations/88adb8b462df2c473b421f617b086dd65351532f28e81a7a1052e1f24da87dea/round-0001.json",
"call_sha256": "0db1c3697d2502946077a94ccf7eb93b9d0fe6b84312023c55f0aaba64b18826",
"phase": "INTERPRETING_CASE",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [],
"contributes_to": [],
"group": "entity_resolution",
"prompt_version": "entity_resolution_assessment_v4",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 6.0,
"usage": {
"input_tokens": 25738,
"output_tokens": 471,
"reasoning_tokens": null,
"cached_input_tokens": null,
"uncached_input_tokens": null,
"visible_output_tokens": null
}
},
{
"call_ref": "llm/operations/572afccbd4034d9c5485045cb4bb28b61fdd40e48dc8a3fab67cfc17e997cd65/round-0001.json",
"call_sha256": "0f2b7690a9b10e46bee94bccd19cd7b75a42feec34f5f6d113edb26b62d0b971",
"phase": "INTERPRETING_CASE",
"status": "SUCCESS",
"code": null,
"validation_errors": [
"INTERPRETATION_SOURCE_KIND"
],
"accepted_by": [],
"contributes_to": [
"case_interpretation",
"hypotheses"
],
"group": "planning",
"prompt_version": "case_plan_v2",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 39.672,
"usage": {
"input_tokens": 13886,
"output_tokens": 5878,
"reasoning_tokens": 3057,
"cached_input_tokens": null,
"uncached_input_tokens": null,
"visible_output_tokens": 2821
}
},
{
"call_ref": "llm/operations/cdf9fddd5dccf7c6859f2f4c5aa65e46778d6d44e446cf579fa6b6f462b9b85c/round-0001.json",
"call_sha256": "b085af1baf09f197207746c601ba9f3962650d3edc3041fade197bf13b64ea35",
"phase": "INTERPRETING_CASE",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [
"case_interpretation",
"hypotheses"
],
"contributes_to": [
"case_interpretation",
"hypotheses"
],
"group": "planning",
"prompt_version": "case_plan_statement_repair_v2",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 39.25,
"usage": {
"input_tokens": 7466,
"output_tokens": 6209,
"reasoning_tokens": 5949,
"cached_input_tokens": null,
"uncached_input_tokens": null,
"visible_output_tokens": 260
}
},
{
"call_ref": "llm/operations/ca6cdff81de3cbae0492c26802daa287b11bf4fabefad477a9b64f1b6db30112/round-0001.json",
"call_sha256": "098318f88855084294844cf53f0e496e781251aee4b877ca67f4e834b6027e19",
"phase": "INVESTIGATING_HYPOTHESES",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [],
"contributes_to": [],
"group": "startup_queries",
"prompt_version": "investigation_startup_v1",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 7.969,
"usage": {
"input_tokens": 8168,
"output_tokens": 656,
"reasoning_tokens": 550,
"cached_input_tokens": 7851,
"uncached_input_tokens": 317,
"visible_output_tokens": 106
}
},
{
"call_ref": "llm/operations/54ef9b495e10fe49a2ffad5909c89efdbfe7ef7645b601865d12e5223b0651bc/round-0001.json",
"call_sha256": "b000f97d74f9cab3c117acc3c8b9c1c71f8ed5f5bf1c19acfa7289b0a16a6869",
"phase": "INVESTIGATING_HYPOTHESES",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [],
"contributes_to": [],
"group": "startup_queries",
"prompt_version": "investigation_startup_v1",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 27.891,
"usage": {
"input_tokens": 11394,
"output_tokens": 3528,
"reasoning_tokens": 3415,
"cached_input_tokens": 7851,
"uncached_input_tokens": 3543,
"visible_output_tokens": 113
}
},
{
"call_ref": "llm/operations/71fd77c20b2af9bb2d842927c7a39cc8e1cf3e15541f387a26054fdb1047b6e4/round-0001.json",
"call_sha256": "d757391eb9bb08ee868ae017a9bc0f091cc80cf87b7d888af4a69d5110920894",
"phase": "INVESTIGATING_HYPOTHESES",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [],
"contributes_to": [],
"group": "startup_queries",
"prompt_version": "investigation_startup_v1",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 27.953,
"usage": {
"input_tokens": 13525,
"output_tokens": 4324,
"reasoning_tokens": 4242,
"cached_input_tokens": 7851,
"uncached_input_tokens": 5674,
"visible_output_tokens": 82
}
},
{
"call_ref": "llm/operations/0270771b422d04af95c20c38558476ded170af96569985af436406482d8d043a/round-0001.json",
"call_sha256": "15fc914607b6316571d7ec23dfa3586dddff0d4953a6bb7efd860bdfb96dc4af",
"phase": "INVESTIGATING_HYPOTHESES",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [
"startup_review_H1",
"startup_review_H2",
"startup_review_H3"
],
"contributes_to": [
"startup_review_H1",
"startup_review_H2",
"startup_review_H3"
],
"group": "startup_joint_review",
"prompt_version": "investigation_startup_review_v1",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 22.204,
"usage": {
"input_tokens": 9442,
"output_tokens": 2953,
"reasoning_tokens": 2239,
"cached_input_tokens": 2060,
"uncached_input_tokens": 7382,
"visible_output_tokens": 714
}
},
{
"call_ref": "llm/operations/c619a7901de5674f54e8371759fba596a7746f67d62499b198780813912e5c31/round-0001.json",
"call_sha256": "eb5809d3035ca00eeb51403ff2b1c56e1bd0020f9092369c101704e892b76af7",
"phase": "INVESTIGATING_HYPOTHESES",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [],
"contributes_to": [],
"group": "investigation_followup",
"prompt_version": "investigation_project_v20",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 6.813,
"usage": {
"input_tokens": 7704,
"output_tokens": 423,
"reasoning_tokens": 329,
"cached_input_tokens": 6077,
"uncached_input_tokens": 1627,
"visible_output_tokens": 94
}
},
{
"call_ref": "llm/operations/b5de43dfa1122fa86decfa538047ee813621839eded8ed15356d341f3439e383/round-0001.json",
"call_sha256": "6b73f365694e2a4cb8e745fd2c82025ee0309d3a317151377e803a3aa06d5f3e",
"phase": "INVESTIGATING_HYPOTHESES",
"status": "SUCCESS",
"code": null,
"validation_errors": [],
"accepted_by": [],
"contributes_to": [],
"group": "investigation_followup",
"prompt_version": "investigation_project_v20",
"provider": "gemini-native",
"model": "gemini-3.8-flash",
"config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69",
"elapsed_seconds": 7.297,
"usage": {
"input_tokens": 20088,
"output_tokens": 886,
"reasoning_tokens": 675,
"cached_input_tokens": 6077,
"uncached_input_tokens": 14011,
"visible_output_tokens": 211
}
}
]
}
docs/test-results/token-usage/comparison-startup-batch-20261020-g00000067.json
+83
View File
@@ -0,0 +1,83 @@
{
"date": "2026-10-20",
"baseline": "bug20-joint-planning-20261019.json",
"candidate": "bug20-startup-batch-20261020-g00000067.json",
"old_three_verbose_starts": {
"call_count": 3,
"input_tokens": {
"known_sum": 23929,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 12476,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 11263,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 1213,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 15375,
"missing_call_count": 0
},
"api_elapsed_seconds_sum": 125.891
},
"new_three_lean_queries": {
"call_count": 3,
"input_tokens": {
"known_sum": 33087,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 8508,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 8207,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 301,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 23553,
"missing_call_count": 0
},
"api_elapsed_seconds_sum": 63.813
},
"new_queries_plus_joint_review": {
"call_count": 4,
"input_tokens": {
"known_sum": 42529,
"missing_call_count": 0
},
"output_tokens": {
"known_sum": 11461,
"missing_call_count": 0
},
"reasoning_tokens": {
"known_sum": 10446,
"missing_call_count": 0
},
"visible_output_tokens": {
"known_sum": 1015,
"missing_call_count": 0
},
"cached_input_tokens": {
"known_sum": 25613,
"missing_call_count": 0
},
"api_elapsed_seconds_sum": 86.017
},
"limitations": [
"Different individual runs and different queries, not a controlled paired benchmark",
"The new joint review settles all initial results; the three old startup calls left own result assessments for subsequent loop calls",
"Includes joint review cost; excludes planning, name resolution, later investigation and any final diagnosis",
"No final euro cost reported yet; input/cache/storage pricing must be considered separately"
]
}