From 3beac3bd1c372103de0af79fb67a122b469c813e Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Tue, 20 Oct 2026 10:59:33 +0200 Subject: [PATCH] Archive standalone token usage reports and retention rules --- AGENTS.md | 7 + docs/test-results/token-usage/README.md | 20 + .../bug20-joint-planning-20261019.json | 508 +++++++++++++++++ .../bug20-joint-recovery-20261020.json | 216 +++++++ .../bug20-joint-structure-20261020.json | 213 +++++++ .../token-usage/bug20-joint-v2-20261020.json | 218 +++++++ .../bug20-shared-recovery-20261019.json | 530 ++++++++++++++++++ ...comparison-separate-vs-joint-20261020.json | 117 ++++ 8 files changed, 1829 insertions(+) create mode 100644 docs/test-results/token-usage/README.md create mode 100644 docs/test-results/token-usage/bug20-joint-planning-20261019.json create mode 100644 docs/test-results/token-usage/bug20-joint-recovery-20261020.json create mode 100644 docs/test-results/token-usage/bug20-joint-structure-20261020.json create mode 100644 docs/test-results/token-usage/bug20-joint-v2-20261020.json create mode 100644 docs/test-results/token-usage/bug20-shared-recovery-20261019.json create mode 100644 docs/test-results/token-usage/comparison-separate-vs-joint-20261020.json diff --git a/AGENTS.md b/AGENTS.md index b6b8a09..b562f76 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,3 +15,10 @@ - Kostenpflichtige LLM-Testläufe führt der Nutzer selbst aus, sofern er nicht ausdrücklich eine andere Ausführung beauftragt. - Lokale Commits sind erlaubt. Nicht pushen; der Nutzer führt Pushes selbst durch. - `docs/architecture.md` ist die maßgebliche Designspezifikation und darf ohne ausdrücklichen Auftrag nicht geändert werden. + +## Dauerhafte Testmetriken + +- Nach jeder ausgewerteten LLM-Testausführung die Tokenbilanz unabhängig vom Case-Paket unter `docs/test-results/token-usage/` sichern. Vor dem Entfernen eines Test-Workspaces das Vorhandensein des Berichts prüfen. +- Gesamtverbrauch einschließlich Korrektur-/Wiederholungsaufrufen und bereinigte Phasenbilanzen getrennt ausweisen; gemeinsame Aufrufe nur einmal zählen. Fehlende Werte bleiben als unbekannt gekennzeichnet. Denktokens nicht erneut zu Ausgabetokens addieren, wenn sie darin bereits enthalten sind. +- Lauf-/Case-Identität, Ergebnis und Messumfang, Prompt-/Modellversionen, Konfigurations-/Quellenhashes, Cachewerte, verfügbare API-/Falllaufzeiten sowie Kostenstände mit Abrechnungsstatus festhalten. Bereits berichtete Snapshots erhalten, spätere Fortschritte als neue Snapshots ergänzen. +- Berichte enthalten keine Prompts, vollständigen Antworten, Secrets oder Konfigurationsinhalte. Die kleinen Metrikberichte und Vergleiche dürfen langfristig versioniert bleiben; sie zählen zum Speicherbudget für Testdaten. diff --git a/docs/test-results/token-usage/README.md b/docs/test-results/token-usage/README.md new file mode 100644 index 0000000..6cc4552 --- /dev/null +++ b/docs/test-results/token-usage/README.md @@ -0,0 +1,20 @@ +# Dauerhafte Tokenbilanzen + +Diese kleinen JSON-Berichte bleiben unabhängig von den Testpaketen versioniert erhalten. +Sie enthalten Einzelaufrufe, Prompt-/Modellversion, Konfigurationshash, Quellenhashes, +Lifecycle-Ergebnis sowie Gesamt- und bereinigte Planungsbilanzen. Keine Prompts, +Antworttexte oder Zugangsdaten. Ein gemeinsamer Aufruf wird nur einmal gezählt. + +`output_tokens` enthält bei diesen Providermetriken bereits die Denktokens. +Fehlende Messwerte bleiben null; Summen weisen die Anzahl fehlender Messwerte aus. +API-Zeitsummen sind keine gesamte Falllaufzeit. Annahme/Ablehnung wird nur bei +vorliegenden Artefaktverweisen beziehungsweise Validierungsfehlern ausgewiesen; +fehlende Zuordnung bedeutet nicht automatisch einen erfolgreichen Aufruf. + +Nach jeder Testauswertung einen neuen Bericht beziehungsweise einen eindeutig datierten +Snapshot anlegen; vorhandene Messstände nicht mit späteren Fortschritten überschreiben. +Kontostände mit Beginn, Zwischenstand und endgültigem Stand samt Abrechnungsverzögerung +festhalten. Vor dem Löschen eines Test-Workspaces das Vorhandensein der Bilanz prüfen. + +Der Vergleich `comparison-separate-vs-joint-20261020.json` sichert den im Chat +besprochenen Vergleich. Der aktuelle Kostenstand 12,47 EUR ist noch vorläufig. diff --git a/docs/test-results/token-usage/bug20-joint-planning-20261019.json b/docs/test-results/token-usage/bug20-joint-planning-20261019.json new file mode 100644 index 0000000..7128ea9 --- /dev/null +++ b/docs/test-results/token-usage/bug20-joint-planning-20261019.json @@ -0,0 +1,508 @@ +{ + "format": "viano-token-usage-report", + "format_version": 1, + "recorded_date": "2026-10-20", + "run": "bug20-joint-planning-20261019", + "case_id": "BUG-000020", + "package_id": "trial-joint-planning-BUG-000020-20261019", + "package_generation": 97, + "analysis_revision": 1, + "state": "OUTPUT_READY", + "routing": { + "destination": "outbox", + "reason": null, + "retry_count": 0, + "retry_not_before": null + }, + "source_package": "service-data/trials/bug20-joint-planning-20261019/cases/897a914b67a02e6109a6b567e377e43c72b90ab216bfb9d2dc24485a5a55db8d/60d94dc476555860dafbb6b852fa5a4d763ea432e1f62723280817bafa379d35/g00000097.zip", + "manifest_sha256": "7ac099c9388bd4fc3fa4e929a3d58a0b8ab79f6b1121999cf596f08db83e17ad", + "notes": [ + "output_tokens includes reasoning_tokens; never add reasoning to output again", + "Null usage is unavailable, not zero; known sums carry missing counts", + "Accepted planning deduplicates calls used by both phase artifacts", + "Elapsed API sums are not wall-clock trial duration", + "Investigation calls without phase-artifact pointers are not automatically classified as accepted or rejected" + ], + "totals_all_calls": { + "call_count": 18, + "input_tokens": { + "known_sum": 254110, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 48571, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 33711, + "missing_call_count": 4 + }, + "visible_output_tokens": { + "known_sum": 13622, + "missing_call_count": 4 + }, + "cached_input_tokens": { + "known_sum": 50576, + "missing_call_count": 8 + }, + "api_elapsed_seconds_sum": 393.686 + }, + "totals_planning_all_attempts": { + "call_count": 3, + "input_tokens": { + "known_sum": 44531, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 16011, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 8854, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 7157, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 3 + }, + "api_elapsed_seconds_sum": 106.218 + }, + "totals_planning_accepted_only": { + "call_count": 2, + "input_tokens": { + "known_sum": 31545, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 9282, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 5714, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 3568, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 2 + }, + "api_elapsed_seconds_sum": 61.609 + }, + "calls": [ + { + "call_ref": "llm/operations/6c0a36f93bea41b87241a30769bd92bb6dc387869b46d3a013ec595cfc7e9548/round-0001.json", + "call_sha256": "ea18fbf657b95ee8e52e35062d8aced37fa9204d7bcc8437af9bb9766728099d", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_plan_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 2.985, + "usage": { + "input_tokens": 21005, + "output_tokens": 180, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/88adb8b462df2c473b421f617b086dd65351532f28e81a7a1052e1f24da87dea/round-0001.json", + "call_sha256": "f4e388bb58162ae651534684aeedc8894892dbc4fbcd731d4a997d2e37b4d969", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_assessment_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 5.078, + "usage": { + "input_tokens": 25764, + "output_tokens": 501, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/572afccbd4034d9c5485045cb4bb28b61fdd40e48dc8a3fab67cfc17e997cd65/round-0001.json", + "call_sha256": "81b5649c121cf83dc049bac836a5fd607aed4031bf1626f6c0768e1ff19f2fa5", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [ + "INTERPRETATION_SOURCE_KIND" + ], + "accepted_by": [], + "prompt_version": "case_plan_v1", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 44.609, + "usage": { + "input_tokens": 12986, + "output_tokens": 6729, + "reasoning_tokens": 3140, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 3589 + } + }, + { + "call_ref": "llm/operations/cdf9fddd5dccf7c6859f2f4c5aa65e46778d6d44e446cf579fa6b6f462b9b85c/round-0001.json", + "call_sha256": "640befa42a34ed697eb234d2be12c43033d6aaa78f7c92cc94c9619f8adbfe81", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "case_interpretation" + ], + "prompt_version": "case_plan_interpretation_repair_v1", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 26.859, + "usage": { + "input_tokens": 15650, + "output_tokens": 4446, + "reasoning_tokens": 2526, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1920 + } + }, + { + "call_ref": "llm/operations/5fa06d3e4f4063a74b5818fdba690a7e1cd8b145506fb7c1d2a5deefc421ef98/round-0001.json", + "call_sha256": "a91fc7ebdb21be2bce1281b8d5393371a5483aa2edbdce21488946e1a0894a81", + "phase": "GENERATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "hypotheses" + ], + "prompt_version": "hypotheses_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 34.75, + "usage": { + "input_tokens": 15895, + "output_tokens": 4836, + "reasoning_tokens": 3188, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1648 + } + }, + { + "call_ref": "llm/operations/ca6cdff81de3cbae0492c26802daa287b11bf4fabefad477a9b64f1b6db30112/round-0001.json", + "call_sha256": "b938dc074fda1a8fa5a4aa4d9d4b78c76f7b74d38a28ae2ff433f2ad0b71502e", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 11.125, + "usage": { + "input_tokens": 6028, + "output_tokens": 1184, + "reasoning_tokens": 844, + "cached_input_tokens": 5125, + "uncached_input_tokens": 903, + "visible_output_tokens": 340 + } + }, + { + "call_ref": "llm/operations/54ef9b495e10fe49a2ffad5909c89efdbfe7ef7645b601865d12e5223b0651bc/round-0001.json", + "call_sha256": "39103cc7bfda62cc8956d49d379394f5a23c515219de85d9fe255f1ee5970e50", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 44.469, + "usage": { + "input_tokens": 8448, + "output_tokens": 6491, + "reasoning_tokens": 5965, + "cached_input_tokens": 5125, + "uncached_input_tokens": 3323, + "visible_output_tokens": 526 + } + }, + { + "call_ref": "llm/operations/71fd77c20b2af9bb2d842927c7a39cc8e1cf3e15541f387a26054fdb1047b6e4/round-0001.json", + "call_sha256": "8708f97fc082cd4ee6eaa1597bb07e4310a64e60ee58cc23fcfecb22f47eef53", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 70.297, + "usage": { + "input_tokens": 9453, + "output_tokens": 4801, + "reasoning_tokens": 4454, + "cached_input_tokens": 5125, + "uncached_input_tokens": 4328, + "visible_output_tokens": 347 + } + }, + { + "call_ref": "llm/operations/c619a7901de5674f54e8371759fba596a7746f67d62499b198780813912e5c31/round-0001.json", + "call_sha256": "43e2a484b19dac9bb4f53f8ddafd6d2d5fe4d4c1291f93e32d519ff401ce7ff4", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 11.203, + "usage": { + "input_tokens": 13739, + "output_tokens": 1247, + "reasoning_tokens": 1045, + "cached_input_tokens": 6274, + "uncached_input_tokens": 7465, + "visible_output_tokens": 202 + } + }, + { + "call_ref": "llm/operations/b5de43dfa1122fa86decfa538047ee813621839eded8ed15356d341f3439e383/round-0001.json", + "call_sha256": "4e0e2872ef8b331c69d0d9435b1469b200e41b10ac77cbb24da84c49024def2d", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 6.359, + "usage": { + "input_tokens": 15338, + "output_tokens": 711, + "reasoning_tokens": 541, + "cached_input_tokens": 6274, + "uncached_input_tokens": 9064, + "visible_output_tokens": 170 + } + }, + { + "call_ref": "llm/operations/6849c974442865e5ca3e8f146ef4520c275f2e37a25fb7caa498f17089ed6dc1/round-0001.json", + "call_sha256": "ef958be97b1cc0217a431a6945f3cdf282bc079392ce379c19279e8ca5fffc4f", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 13.015, + "usage": { + "input_tokens": 13963, + "output_tokens": 1560, + "reasoning_tokens": 1272, + "cached_input_tokens": 6274, + "uncached_input_tokens": 7689, + "visible_output_tokens": 288 + } + }, + { + "call_ref": "llm/operations/475d5f458554c35131f24fc92fb2e762c69a8d9c6d4ab9d6426fbd72e6303af3/round-0001.json", + "call_sha256": "1e912ee014b2af06b6105be3e9c28940f5e73b37fb8fe9720597d67db19e3671", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 13.093, + "usage": { + "input_tokens": 14745, + "output_tokens": 1485, + "reasoning_tokens": 1208, + "cached_input_tokens": 6274, + "uncached_input_tokens": 8471, + "visible_output_tokens": 277 + } + }, + { + "call_ref": "llm/operations/8f58f73379896f4edda417f564668be151dea6f1dd75d1872c8aeb64d0cef5ce/round-0001.json", + "call_sha256": "7b101261a0613dec0cb0cc531ca5298b531853e2987087e27204a51c6a6367ed", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 23.562, + "usage": { + "input_tokens": 16348, + "output_tokens": 2930, + "reasoning_tokens": 2461, + "cached_input_tokens": 4469, + "uncached_input_tokens": 11879, + "visible_output_tokens": 469 + } + }, + { + "call_ref": "llm/operations/a4ed5334e4125ed14bcbca18629a561513bba48ef92afecc570c14d9e90e05cc/round-0001.json", + "call_sha256": "442d679bccdde15911ed18d2c0a620a911d55d825ea53855f39b7f704aec70fe", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 5.391, + "usage": { + "input_tokens": 6521, + "output_tokens": 336, + "reasoning_tokens": null, + "cached_input_tokens": 2818, + "uncached_input_tokens": 3703, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/b7f2e84e7ce1ab15fea771e71ef5c71f3321d1b419eefa7f3f69f110b5c356c6/round-0001.json", + "call_sha256": "a11d6b43f076342897d1e1485fc3bb53ac3b8db7ac0b7ffb1347b78f46d2945f", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 3.719, + "usage": { + "input_tokens": 7431, + "output_tokens": 221, + "reasoning_tokens": null, + "cached_input_tokens": 2818, + "uncached_input_tokens": 4613, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/25febf70da1f602d9d830c61c59e4504617a8bc05beac14676d7fd4e9f6589f9/round-0001.json", + "call_sha256": "0e3dfa99c65c75b58fe6553db44b578b86fe2ef569caa9257e67f4d498aa65f6", + "phase": "BUILDING_DIAGNOSIS", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "diagnosis" + ], + "prompt_version": "diagnosis_early_v7", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 25.672, + "usage": { + "input_tokens": 21736, + "output_tokens": 3514, + "reasoning_tokens": 2300, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1214 + } + }, + { + "call_ref": "llm/operations/17917402a4c04bd20ae3b0a2f256a39daa4c026a0afff5f80f408478611d4e03/round-0001.json", + "call_sha256": "267554a74192630ddabdcd221cf78e62067b58abce18bf5403e9fcf061bfe74d", + "phase": "BUILDING_DIAGNOSIS", + "status": "ERROR", + "code": "LLM_RESPONSE", + "validation_errors": [ + "LLM_RESPONSE" + ], + "accepted_by": [], + "prompt_version": "change_request_v8", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 26.75, + "usage": { + "input_tokens": 14456, + "output_tokens": 3815, + "reasoning_tokens": 2497, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1318 + } + }, + { + "call_ref": "llm/operations/c06b231c8faccaeb58e9684b7d28a566e8bd629c8dcd13a423c9ba2a012d27b6/round-0001.json", + "call_sha256": "0f6319fbe2b0defd9bcf0ad6c3ba9ff2ce81e8dcd6750e052552b29158c28d2b", + "phase": "BUILDING_DIAGNOSIS", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "change_request" + ], + "prompt_version": "change_request_v8", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 24.75, + "usage": { + "input_tokens": 14604, + "output_tokens": 3584, + "reasoning_tokens": 2270, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1314 + } + } + ] +} diff --git a/docs/test-results/token-usage/bug20-joint-recovery-20261020.json b/docs/test-results/token-usage/bug20-joint-recovery-20261020.json new file mode 100644 index 0000000..c246c58 --- /dev/null +++ b/docs/test-results/token-usage/bug20-joint-recovery-20261020.json @@ -0,0 +1,216 @@ +{ + "format": "viano-token-usage-report", + "format_version": 1, + "recorded_date": "2026-10-20", + "run": "bug20-joint-recovery-20261020", + "case_id": "BUG-000020", + "package_id": "trial-joint-recovery-BUG-000020-20261020", + "package_generation": 22, + "analysis_revision": 1, + "state": "FAILED_TO_ANALYZE", + "routing": { + "destination": "review", + "reason": "MODEL_VALIDATION_EXHAUSTED", + "retry_count": 0, + "retry_not_before": null + }, + "source_package": "service-data/trials/bug20-joint-recovery-20261020/cases/897a914b67a02e6109a6b567e377e43c72b90ab216bfb9d2dc24485a5a55db8d/307269a3ff8069115c563dc25fdb9a5f1762815d5e539d78b1c43fa8d6ab73f3/g00000022.zip", + "manifest_sha256": "91010fcbd8556b45af67795d9344cc14e93f57ad273db5028cc7fdcb25b971fc", + "notes": [ + "output_tokens includes reasoning_tokens; never add reasoning to output again", + "Null usage is unavailable, not zero; known sums carry missing counts", + "Accepted planning deduplicates calls used by both phase artifacts", + "Elapsed API sums are not wall-clock trial duration", + "Investigation calls without phase-artifact pointers are not automatically classified as accepted or rejected" + ], + "totals_all_calls": { + "call_count": 5, + "input_tokens": { + "known_sum": 81023, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 24850, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 19307, + "missing_call_count": 2 + }, + "visible_output_tokens": { + "known_sum": 4858, + "missing_call_count": 2 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 5 + }, + "api_elapsed_seconds_sum": 159.578 + }, + "totals_planning_all_attempts": { + "call_count": 3, + "input_tokens": { + "known_sum": 34252, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 24165, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 19307, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 4858, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 3 + }, + "api_elapsed_seconds_sum": 151.609 + }, + "totals_planning_accepted_only": { + "call_count": 0, + "input_tokens": { + "known_sum": 0, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 0, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 0, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 0, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 0 + }, + "api_elapsed_seconds_sum": 0 + }, + "calls": [ + { + "call_ref": "llm/operations/6c0a36f93bea41b87241a30769bd92bb6dc387869b46d3a013ec595cfc7e9548/round-0001.json", + "call_sha256": "0e346b87943a4ef5bddeb6f213c95b09cb327c983d99c73937adb43ed7d2a1a2", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_plan_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 3.094, + "usage": { + "input_tokens": 21006, + "output_tokens": 180, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/88adb8b462df2c473b421f617b086dd65351532f28e81a7a1052e1f24da87dea/round-0001.json", + "call_sha256": "10dc5ce6bbb9aa952fed8f7e05be7f3ac3c84b1cdf2fac7c7109c6bdca6cd6c4", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_assessment_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 4.875, + "usage": { + "input_tokens": 25765, + "output_tokens": 505, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/572afccbd4034d9c5485045cb4bb28b61fdd40e48dc8a3fab67cfc17e997cd65/round-0001.json", + "call_sha256": "2ff4e170dc33b33d58ea7c44e497dfc10302a8170248b5789abe67d6107fb72c", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [ + "INTERPRETATION_SCHEMA" + ], + "accepted_by": [], + "prompt_version": "case_plan_v2", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 28.234, + "usage": { + "input_tokens": 13918, + "output_tokens": 4466, + "reasoning_tokens": 1602, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 2864 + } + }, + { + "call_ref": "llm/operations/cdf9fddd5dccf7c6859f2f4c5aa65e46778d6d44e446cf579fa6b6f462b9b85c/round-0001.json", + "call_sha256": "4567683fb436d3467c99c720e2721dbf88e9d886927688310daa167c36654bf5", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [ + "INTERPRETATION_EXPECTATION_UNKNOWN" + ], + "accepted_by": [], + "prompt_version": "case_plan_interpretation_repair_v1", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 26.828, + "usage": { + "input_tokens": 14078, + "output_tokens": 4652, + "reasoning_tokens": 2854, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1798 + } + }, + { + "call_ref": "llm/operations/2cd36795173fe6d547ff571814187ce1b12b3a7ec6accd4ebd98869d46bad248/round-0001.json", + "call_sha256": "e749e5545c6403ea2f543674d06d8092490ecfaa80e9c8d74c412f1597cc7768", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [ + "INTERPRETATION_ENTITY_PROVENANCE" + ], + "accepted_by": [], + "prompt_version": "case_plan_statement_repair_v2", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 96.547, + "usage": { + "input_tokens": 6256, + "output_tokens": 15047, + "reasoning_tokens": 14851, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 196 + } + } + ] +} diff --git a/docs/test-results/token-usage/bug20-joint-structure-20261020.json b/docs/test-results/token-usage/bug20-joint-structure-20261020.json new file mode 100644 index 0000000..eb94bbe --- /dev/null +++ b/docs/test-results/token-usage/bug20-joint-structure-20261020.json @@ -0,0 +1,213 @@ +{ + "format": "viano-token-usage-report", + "format_version": 1, + "recorded_date": "2026-10-20", + "run": "bug20-joint-structure-20261020", + "case_id": "BUG-000020", + "package_id": "trial-joint-structure-BUG-000020-20261020", + "package_generation": 37, + "analysis_revision": 1, + "state": "INVESTIGATING_HYPOTHESES", + "routing": { + "destination": "cases", + "reason": null, + "retry_count": 1, + "retry_not_before": null + }, + "source_package": "service-data/trials/bug20-joint-structure-20261020/cases/897a914b67a02e6109a6b567e377e43c72b90ab216bfb9d2dc24485a5a55db8d/07ef748c1c769f3abd169538220ccd6891f2b1d7556bbb7dde562f4f59ac5900/g00000037.zip", + "manifest_sha256": "7dfa4b9a067e2d5b8449408c8504b42e5d7ea69732c2224f09085cee41957cc1", + "notes": [ + "output_tokens includes reasoning_tokens; never add reasoning to output again", + "Null usage is unavailable, not zero; known sums carry missing counts", + "Accepted planning deduplicates calls used by both phase artifacts", + "Elapsed API sums are not wall-clock trial duration", + "Investigation calls without phase-artifact pointers are not automatically classified as accepted or rejected" + ], + "totals_all_calls": { + "call_count": 5, + "input_tokens": { + "known_sum": 73501, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 15281, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 11160, + "missing_call_count": 2 + }, + "visible_output_tokens": { + "known_sum": 3462, + "missing_call_count": 2 + }, + "cached_input_tokens": { + "known_sum": 9528, + "missing_call_count": 3 + }, + "api_elapsed_seconds_sum": 113.545 + }, + "totals_planning_all_attempts": { + "call_count": 1, + "input_tokens": { + "known_sum": 13733, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 7863, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 5247, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 2616, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 1 + }, + "api_elapsed_seconds_sum": 51.156 + }, + "totals_planning_accepted_only": { + "call_count": 1, + "input_tokens": { + "known_sum": 13733, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 7863, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 5247, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 2616, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 1 + }, + "api_elapsed_seconds_sum": 51.156 + }, + "calls": [ + { + "call_ref": "llm/operations/6c0a36f93bea41b87241a30769bd92bb6dc387869b46d3a013ec595cfc7e9548/round-0001.json", + "call_sha256": "bcde1a08152b25e3794dc9b3c0914c0ad3b2cec9e27a376ca627340865388ad0", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_plan_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 3.359, + "usage": { + "input_tokens": 21010, + "output_tokens": 162, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/88adb8b462df2c473b421f617b086dd65351532f28e81a7a1052e1f24da87dea/round-0001.json", + "call_sha256": "d31a5173fa6be1c9e8d5b89a2ac62ea82c8c9d93286a5bac5877e489c89cc0aa", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_assessment_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 5.0, + "usage": { + "input_tokens": 25225, + "output_tokens": 497, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/572afccbd4034d9c5485045cb4bb28b61fdd40e48dc8a3fab67cfc17e997cd65/round-0001.json", + "call_sha256": "6d957252d8ca9075f34e2c0e48bef91c5499debcc6c811abcf6224200e10eaa9", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "case_interpretation", + "hypotheses" + ], + "prompt_version": "case_plan_v2", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 51.156, + "usage": { + "input_tokens": 13733, + "output_tokens": 7863, + "reasoning_tokens": 5247, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 2616 + } + }, + { + "call_ref": "llm/operations/ca6cdff81de3cbae0492c26802daa287b11bf4fabefad477a9b64f1b6db30112/round-0001.json", + "call_sha256": "d7b3082d06cf3e990dea9c793f7c529d6e731a401c7bca8a8ba2310fd603dbf9", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 13.171, + "usage": { + "input_tokens": 5566, + "output_tokens": 1559, + "reasoning_tokens": 1213, + "cached_input_tokens": 4764, + "uncached_input_tokens": 802, + "visible_output_tokens": 346 + } + }, + { + "call_ref": "llm/operations/54ef9b495e10fe49a2ffad5909c89efdbfe7ef7645b601865d12e5223b0651bc/round-0001.json", + "call_sha256": "ff3132842acdfb2800c7865c791a2ae78713611ada8fda5b26258de56f19280b", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 40.859, + "usage": { + "input_tokens": 7967, + "output_tokens": 5200, + "reasoning_tokens": 4700, + "cached_input_tokens": 4764, + "uncached_input_tokens": 3203, + "visible_output_tokens": 500 + } + } + ] +} diff --git a/docs/test-results/token-usage/bug20-joint-v2-20261020.json b/docs/test-results/token-usage/bug20-joint-v2-20261020.json new file mode 100644 index 0000000..b06f838 --- /dev/null +++ b/docs/test-results/token-usage/bug20-joint-v2-20261020.json @@ -0,0 +1,218 @@ +{ + "format": "viano-token-usage-report", + "format_version": 1, + "recorded_date": "2026-10-20", + "run": "bug20-joint-v2-20261020", + "case_id": "BUG-000020", + "package_id": "trial-joint-v2-BUG-000020-20261020", + "package_generation": 25, + "analysis_revision": 1, + "state": "GENERATING_HYPOTHESES", + "routing": { + "destination": "cases", + "reason": null, + "retry_count": 0, + "retry_not_before": null + }, + "source_package": "service-data/trials/bug20-joint-v2-20261020/cases/897a914b67a02e6109a6b567e377e43c72b90ab216bfb9d2dc24485a5a55db8d/1825790dc2ab9100da82a89d884cd25d31439b888bb7db99aea8f551d9ce65c6/g00000025.zip", + "manifest_sha256": "98e913ceb230a9babc8ada4ad00db896609dce9322b996093ed302794dcf72cf", + "notes": [ + "output_tokens includes reasoning_tokens; never add reasoning to output again", + "Null usage is unavailable, not zero; known sums carry missing counts", + "Accepted planning deduplicates calls used by both phase artifacts", + "Elapsed API sums are not wall-clock trial duration", + "Investigation calls without phase-artifact pointers are not automatically classified as accepted or rejected" + ], + "totals_all_calls": { + "call_count": 5, + "input_tokens": { + "known_sum": 82295, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 16520, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 9624, + "missing_call_count": 2 + }, + "visible_output_tokens": { + "known_sum": 6341, + "missing_call_count": 2 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 5 + }, + "api_elapsed_seconds_sum": 113.108 + }, + "totals_planning_all_attempts": { + "call_count": 3, + "input_tokens": { + "known_sum": 36118, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 15965, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 9624, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 6341, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 3 + }, + "api_elapsed_seconds_sum": 106.187 + }, + "totals_planning_accepted_only": { + "call_count": 2, + "input_tokens": { + "known_sum": 22495, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 10075, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 7763, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 2312, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 2 + }, + "api_elapsed_seconds_sum": 68.765 + }, + "calls": [ + { + "call_ref": "llm/operations/6c0a36f93bea41b87241a30769bd92bb6dc387869b46d3a013ec595cfc7e9548/round-0001.json", + "call_sha256": "5b082a63e935f68306a691452e5f59c21aa8bd4c621be1431748869de36de836", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_plan_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 2.734, + "usage": { + "input_tokens": 21009, + "output_tokens": 134, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/88adb8b462df2c473b421f617b086dd65351532f28e81a7a1052e1f24da87dea/round-0001.json", + "call_sha256": "9536197fbdb90d354a08c307434fad54ad5b1e50684fc42ec6284c36d670293c", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_assessment_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 4.187, + "usage": { + "input_tokens": 25168, + "output_tokens": 421, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/572afccbd4034d9c5485045cb4bb28b61fdd40e48dc8a3fab67cfc17e997cd65/round-0001.json", + "call_sha256": "07eb2df391d4abca1c85733f8fc4f2559dcc3c71033c92022cfc70b3812e4f94", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [ + "INTERPRETATION_SCHEMA" + ], + "accepted_by": [], + "prompt_version": "case_plan_v2", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 37.422, + "usage": { + "input_tokens": 13623, + "output_tokens": 5890, + "reasoning_tokens": 1861, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 4029 + } + }, + { + "call_ref": "llm/operations/cdf9fddd5dccf7c6859f2f4c5aa65e46778d6d44e446cf579fa6b6f462b9b85c/round-0001.json", + "call_sha256": "c2ea0441d010892218604a27cb1246bdcf1691f31c228f811e272db8e76fc7c9", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [ + "HYPOTHESIS_PREMISE_SOURCE" + ], + "accepted_by": [ + "case_interpretation" + ], + "prompt_version": "case_plan_statement_repair_v2", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 39.812, + "usage": { + "input_tokens": 7116, + "output_tokens": 6023, + "reasoning_tokens": 5269, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 754 + } + }, + { + "call_ref": "llm/operations/b58ae687fdde681ed31f46a21147298ced592a1fb391d86a3f346bfd3e648096/round-0001.json", + "call_sha256": "8347b755c200a36efa80bafcb4100bf812de861d4589eef9a19619a738e80e94", + "phase": "GENERATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "hypotheses" + ], + "prompt_version": "hypotheses_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 28.953, + "usage": { + "input_tokens": 15379, + "output_tokens": 4052, + "reasoning_tokens": 2494, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1558 + } + } + ] +} diff --git a/docs/test-results/token-usage/bug20-shared-recovery-20261019.json b/docs/test-results/token-usage/bug20-shared-recovery-20261019.json new file mode 100644 index 0000000..8694983 --- /dev/null +++ b/docs/test-results/token-usage/bug20-shared-recovery-20261019.json @@ -0,0 +1,530 @@ +{ + "format": "viano-token-usage-report", + "format_version": 1, + "recorded_date": "2026-10-20", + "run": "bug20-shared-recovery-20261019", + "case_id": "BUG-000020", + "package_id": "trial-shared-recovery-BUG-000020-20261019", + "package_generation": 100, + "analysis_revision": 1, + "state": "OUTPUT_READY", + "routing": { + "destination": "outbox", + "reason": null, + "retry_count": 0, + "retry_not_before": null + }, + "source_package": "service-data/trials/bug20-shared-recovery-20261019/cases/897a914b67a02e6109a6b567e377e43c72b90ab216bfb9d2dc24485a5a55db8d/49c833d7194c687b04596c9b9a4bd0f4f021f54171839fb8336fc85296f98ed6/g00000100.zip", + "manifest_sha256": "849c54c0324094dfa0ed61b28802bebf27a29dadd8c15674013689ebf9b1ba15", + "notes": [ + "output_tokens includes reasoning_tokens; never add reasoning to output again", + "Null usage is unavailable, not zero; known sums carry missing counts", + "Accepted planning deduplicates calls used by both phase artifacts", + "Elapsed API sums are not wall-clock trial duration", + "Investigation calls without phase-artifact pointers are not automatically classified as accepted or rejected" + ], + "totals_all_calls": { + "call_count": 19, + "input_tokens": { + "known_sum": 295146, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 63527, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 49718, + "missing_call_count": 4 + }, + "visible_output_tokens": { + "known_sum": 12441, + "missing_call_count": 4 + }, + "cached_input_tokens": { + "known_sum": 69687, + "missing_call_count": 7 + }, + "api_elapsed_seconds_sum": 509.249 + }, + "totals_planning_all_attempts": { + "call_count": 4, + "input_tokens": { + "known_sum": 63120, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 19587, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 12903, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 6684, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 14024, + "missing_call_count": 3 + }, + "api_elapsed_seconds_sum": 162.937 + }, + "totals_planning_accepted_only": { + "call_count": 2, + "input_tokens": { + "known_sum": 31170, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 11757, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 8328, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 3429, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 14024, + "missing_call_count": 1 + }, + "api_elapsed_seconds_sum": 97.422 + }, + "calls": [ + { + "call_ref": "llm/operations/6c0a36f93bea41b87241a30769bd92bb6dc387869b46d3a013ec595cfc7e9548/round-0001.json", + "call_sha256": "705adf20ab5efe08abfe681e41e988038f7967b230eb485e235bb1f5c60b2e5f", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_plan_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 3.547, + "usage": { + "input_tokens": 21008, + "output_tokens": 194, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/88adb8b462df2c473b421f617b086dd65351532f28e81a7a1052e1f24da87dea/round-0001.json", + "call_sha256": "f1b86b6be890ad059d389f6dfe790da972a34664076b9d3a71a50f6b8adc5e54", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "entity_resolution_assessment_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 5.656, + "usage": { + "input_tokens": 25795, + "output_tokens": 521, + "reasoning_tokens": null, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/572afccbd4034d9c5485045cb4bb28b61fdd40e48dc8a3fab67cfc17e997cd65/round-0001.json", + "call_sha256": "66bf6ec1b261a088dabe25cc1ac9718e502dac35a65ba342ff61d924dcf07f49", + "phase": "INTERPRETING_CASE", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "case_interpretation" + ], + "prompt_version": "case_interpretation_resolved_v8", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 46.922, + "usage": { + "input_tokens": 15109, + "output_tokens": 6992, + "reasoning_tokens": 5155, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1837 + } + }, + { + "call_ref": "llm/operations/5fa06d3e4f4063a74b5818fdba690a7e1cd8b145506fb7c1d2a5deefc421ef98/round-0001.json", + "call_sha256": "bc380985aadade22f74e06b73d0bac5a5bded15078f9fd44884625971fb98d57", + "phase": "GENERATING_HYPOTHESES", + "status": "ERROR", + "code": "LLM_RESPONSE", + "validation_errors": [ + "LLM_RESPONSE" + ], + "accepted_by": [], + "prompt_version": "hypotheses_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 30.094, + "usage": { + "input_tokens": 15889, + "output_tokens": 3578, + "reasoning_tokens": 1963, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1615 + } + }, + { + "call_ref": "llm/operations/b58ae687fdde681ed31f46a21147298ced592a1fb391d86a3f346bfd3e648096/round-0001.json", + "call_sha256": "5fd4b4625d59cfa6409ffe8bd69d3dbb19a2fcc48dd0ffd08a2c2db08718b781", + "phase": "GENERATING_HYPOTHESES", + "status": "ERROR", + "code": "LLM_RESPONSE", + "validation_errors": [ + "LLM_RESPONSE" + ], + "accepted_by": [], + "prompt_version": "hypotheses_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 35.421, + "usage": { + "input_tokens": 16061, + "output_tokens": 4252, + "reasoning_tokens": 2612, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1640 + } + }, + { + "call_ref": "llm/operations/7a2157eed9ff7b1494fcc4bd20246322278721c551c2fa3f2efbc7c6ad88484e/round-0001.json", + "call_sha256": "f290eddf7cf0101f766ae9572dd525d1f7ab5802ce5d690bb064bd232bfc9365", + "phase": "GENERATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "hypotheses" + ], + "prompt_version": "hypotheses_v4", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 50.5, + "usage": { + "input_tokens": 16061, + "output_tokens": 4765, + "reasoning_tokens": 3173, + "cached_input_tokens": 14024, + "uncached_input_tokens": 2037, + "visible_output_tokens": 1592 + } + }, + { + "call_ref": "llm/operations/ca6cdff81de3cbae0492c26802daa287b11bf4fabefad477a9b64f1b6db30112/round-0001.json", + "call_sha256": "a7c442ae8d23276b64163c0837b1bde689e41148a1f8902b5e222aa003aefcb3", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 13.031, + "usage": { + "input_tokens": 6088, + "output_tokens": 1407, + "reasoning_tokens": 1057, + "cached_input_tokens": 5119, + "uncached_input_tokens": 969, + "visible_output_tokens": 350 + } + }, + { + "call_ref": "llm/operations/54ef9b495e10fe49a2ffad5909c89efdbfe7ef7645b601865d12e5223b0651bc/round-0001.json", + "call_sha256": "73cdd99e946df223e0d2473742cd64532a7c4b8bb17d2fe7083f9788e45bb41d", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 47.25, + "usage": { + "input_tokens": 8462, + "output_tokens": 6040, + "reasoning_tokens": 5554, + "cached_input_tokens": 5119, + "uncached_input_tokens": 3343, + "visible_output_tokens": 486 + } + }, + { + "call_ref": "llm/operations/71fd77c20b2af9bb2d842927c7a39cc8e1cf3e15541f387a26054fdb1047b6e4/round-0001.json", + "call_sha256": "a6a08a0d40b10272895171f82bcd4f28dccf9b6f46cead9e34894f64fa57747a", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 101.89, + "usage": { + "input_tokens": 16254, + "output_tokens": 14508, + "reasoning_tokens": 13965, + "cached_input_tokens": 5119, + "uncached_input_tokens": 11135, + "visible_output_tokens": 543 + } + }, + { + "call_ref": "llm/operations/7b8f9ecd4cb00c7e1d8bbd314d9ab15032701b76a0d5cf4256b895312b03a731/round-0001.json", + "call_sha256": "fa50b565159984b7481c306e18eb37cfd2ae23e3a88c666456cc7c294230b41b", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_bootstrap_v23", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 55.703, + "usage": { + "input_tokens": 16350, + "output_tokens": 8197, + "reasoning_tokens": 7739, + "cached_input_tokens": 5119, + "uncached_input_tokens": 11231, + "visible_output_tokens": 458 + } + }, + { + "call_ref": "llm/operations/c619a7901de5674f54e8371759fba596a7746f67d62499b198780813912e5c31/round-0001.json", + "call_sha256": "a8286c3aefd694e23c7ae3920dadbbd8d977ff53c56caa1ef5e8ba73180a48c6", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 9.219, + "usage": { + "input_tokens": 13303, + "output_tokens": 733, + "reasoning_tokens": 489, + "cached_input_tokens": 6271, + "uncached_input_tokens": 7032, + "visible_output_tokens": 244 + } + }, + { + "call_ref": "llm/operations/b5de43dfa1122fa86decfa538047ee813621839eded8ed15356d341f3439e383/round-0001.json", + "call_sha256": "fd33521b4d8adc676dc5030df2b229bd333f244a073bc26215ae6c015a44db65", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 7.454, + "usage": { + "input_tokens": 12258, + "output_tokens": 752, + "reasoning_tokens": 605, + "cached_input_tokens": 6271, + "uncached_input_tokens": 5987, + "visible_output_tokens": 147 + } + }, + { + "call_ref": "llm/operations/6849c974442865e5ca3e8f146ef4520c275f2e37a25fb7caa498f17089ed6dc1/round-0001.json", + "call_sha256": "031712383a580693e1704d204378ddfb43bc38259e8290714f2e0e372a1929c6", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 6.516, + "usage": { + "input_tokens": 13836, + "output_tokens": 611, + "reasoning_tokens": 395, + "cached_input_tokens": 6271, + "uncached_input_tokens": 7565, + "visible_output_tokens": 216 + } + }, + { + "call_ref": "llm/operations/475d5f458554c35131f24fc92fb2e762c69a8d9c6d4ab9d6426fbd72e6303af3/round-0001.json", + "call_sha256": "18826877271bb0f7f96bcc6426bfa7538a2876a6aa6bd7b9dabe6bacff93a29a", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 15.265, + "usage": { + "input_tokens": 14727, + "output_tokens": 1608, + "reasoning_tokens": 1411, + "cached_input_tokens": 6271, + "uncached_input_tokens": 8456, + "visible_output_tokens": 197 + } + }, + { + "call_ref": "llm/operations/8f58f73379896f4edda417f564668be151dea6f1dd75d1872c8aeb64d0cef5ce/round-0001.json", + "call_sha256": "ffe30d21f4f51828fcdac8e3b5b5f9e935f335b66c9ea8f020ff44f905e896ac", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 13.625, + "usage": { + "input_tokens": 14221, + "output_tokens": 1291, + "reasoning_tokens": 904, + "cached_input_tokens": 4463, + "uncached_input_tokens": 9758, + "visible_output_tokens": 387 + } + }, + { + "call_ref": "llm/operations/a4ed5334e4125ed14bcbca18629a561513bba48ef92afecc570c14d9e90e05cc/round-0001.json", + "call_sha256": "0cf4f485d9ff4d0d9573c74d40af4561a8314cda3ce214ec02e9fc1e4b23ecea", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 5.922, + "usage": { + "input_tokens": 15391, + "output_tokens": 301, + "reasoning_tokens": null, + "cached_input_tokens": 2820, + "uncached_input_tokens": 12571, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/b7f2e84e7ce1ab15fea771e71ef5c71f3321d1b419eefa7f3f69f110b5c356c6/round-0001.json", + "call_sha256": "0a7c1d0be70b47185884c534b166b4c7192c952ee44dc5accbcc7f842e8e2b0f", + "phase": "INVESTIGATING_HYPOTHESES", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [], + "prompt_version": "investigation_project_v20", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 4.14, + "usage": { + "input_tokens": 6565, + "output_tokens": 352, + "reasoning_tokens": null, + "cached_input_tokens": 2820, + "uncached_input_tokens": 3745, + "visible_output_tokens": null + } + }, + { + "call_ref": "llm/operations/25febf70da1f602d9d830c61c59e4504617a8bc05beac14676d7fd4e9f6589f9/round-0001.json", + "call_sha256": "c96450d455d354e827206c97915f537e042252e49ae2a107145d79a4889808b2", + "phase": "BUILDING_DIAGNOSIS", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "diagnosis" + ], + "prompt_version": "diagnosis_early_v7", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 24.938, + "usage": { + "input_tokens": 27266, + "output_tokens": 3140, + "reasoning_tokens": 1752, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1388 + } + }, + { + "call_ref": "llm/operations/17917402a4c04bd20ae3b0a2f256a39daa4c026a0afff5f80f408478611d4e03/round-0001.json", + "call_sha256": "7766d97a27a0ee0387f28b18db1f9ee83c86c58e6610626e27dcd28236b6bdc9", + "phase": "BUILDING_DIAGNOSIS", + "status": "SUCCESS", + "code": null, + "validation_errors": [], + "accepted_by": [ + "change_request" + ], + "prompt_version": "change_request_v8", + "provider": "gemini-native", + "model": "gemini-3.8-flash", + "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", + "elapsed_seconds": 32.156, + "usage": { + "input_tokens": 20502, + "output_tokens": 4285, + "reasoning_tokens": 2944, + "cached_input_tokens": null, + "uncached_input_tokens": null, + "visible_output_tokens": 1341 + } + } + ] +} diff --git a/docs/test-results/token-usage/comparison-separate-vs-joint-20261020.json b/docs/test-results/token-usage/comparison-separate-vs-joint-20261020.json new file mode 100644 index 0000000..bd44282 --- /dev/null +++ b/docs/test-results/token-usage/comparison-separate-vs-joint-20261020.json @@ -0,0 +1,117 @@ +{ + "recorded_date": "2026-10-20", + "baseline_report": "bug20-shared-recovery-20261019.json", + "candidate_report": "bug20-joint-structure-20261020.json", + "scope": "interpretation and hypotheses only; excludes entity resolution and investigation", + "accepted_only": { + "baseline": { + "call_count": 2, + "input_tokens": { + "known_sum": 31170, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 11757, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 8328, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 3429, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 14024, + "missing_call_count": 1 + }, + "api_elapsed_seconds_sum": 97.422 + }, + "candidate": { + "call_count": 1, + "input_tokens": { + "known_sum": 13733, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 7863, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 5247, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 2616, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 1 + }, + "api_elapsed_seconds_sum": 51.156 + } + }, + "all_attempts": { + "baseline": { + "call_count": 4, + "input_tokens": { + "known_sum": 63120, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 19587, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 12903, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 6684, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 14024, + "missing_call_count": 3 + }, + "api_elapsed_seconds_sum": 162.937 + }, + "candidate": { + "call_count": 1, + "input_tokens": { + "known_sum": 13733, + "missing_call_count": 0 + }, + "output_tokens": { + "known_sum": 7863, + "missing_call_count": 0 + }, + "reasoning_tokens": { + "known_sum": 5247, + "missing_call_count": 0 + }, + "visible_output_tokens": { + "known_sum": 2616, + "missing_call_count": 0 + }, + "cached_input_tokens": { + "known_sum": 0, + "missing_call_count": 1 + }, + "api_elapsed_seconds_sum": 51.156 + } + }, + "account_eur": { + "candidate_start": 12.46, + "candidate_latest_reported": 12.47, + "candidate_final": null, + "settlement": "pending_provider_delay" + }, + "limitations": [ + "Single runs with different prompt versions, not a controlled paired benchmark", + "Baseline accepted hypothesis call reports 14024 cached input tokens; candidate planning reports no cached count", + "Token reductions do not imply proportional price reductions" + ] +}