Preserve safe LLM failure diagnostics and usage telemetry

docs/llm-failure-diagnostics-v1.md

@@ -0,0 +1,44 @@
+# LLM failure diagnostics
+
+Failed response processing now retains optional `outcome.metadata` and
+`outcome.diagnostics` in the existing v1 `llm/operations/.../round-NNNN.json`
+records. Old records without these fields remain valid and replay without API
+calls. Previously discarded details cannot be recovered retrospectively.
+
+For incomplete replies, diagnostics preserve:
+
+- The API finish reason, restricted to known values; absent/unrecognized values
+  are labeled `missing`/`unknown` rather than copying arbitrary provider text.
+- Boolean refusal/tool-call indicators, not their text or arguments.
+- Input/output tokens and reasoning tokens when supplied by the API.
+- Configured output-token limit, received response bytes and output character count.
+
+Metadata records the configured provider/model, prompt version, elapsed seconds,
+known usage, previous unavailable-provider attempts, and estimated cost only when
+usage and operator-configured prices are available. Missing/invalid token values
+remain unknown, not zero. API output token usage may include reasoning, depending
+on the provider; do not infer reasoning usage when the API omits that field.
+
+JSON parsing errors retain available envelope details. Schema rejection retains
+usage and a safe validation keyword, without storing response contents or the
+validation exception message. Diagnostic fields have their own allowlisted JSON
+schema (`llm-failure-diagnostics-v1.schema.json`). No headers, API keys, provider
+error bodies, refusal text, tool arguments or partial completion text are added.
+
+The same safe details are visible in structured service `llm_round` log events.
+Known usage/cost from failed replies now contributes to existing telemetry
+counters. The connection-test CLI includes diagnostic/metadata fields on errors.
+Existing retry, failover, validation limits, lifecycle and hypothesis budgets are
+unchanged. Successful reply metadata and response contracts remain unchanged.
+
+A `finish_reason = length` indicates an output-budget stop reported by the API;
+compare the output token count with `max_output_tokens`, and inspect reasoning
+usage if available. `content_filter`, refusal/tool-call indicators, or missing
+finish reasons describe different failure conditions. An incomplete response
+alone does not prove a transport timeout. Transport/availability failures still
+use their existing codes and HTTP attempt records; usage may be unavailable.
+
+Tests use local HTTP fixtures and model fixtures. They cover truncation,
+refusal/tool calls, malformed usage, unknown finish reasons, JSON/schema failures,
+usage persistence, replay without network calls, telemetry counters/logs and
+removal of free provider text. No paid model request is needed.

src/viano_bug_analysis/connection_test.py

@@ -21,7 +21,12 @@ async def probe(gateway):
     except GatewayUnavailable as error:
         return {"status": "UNAVAILABLE", "attempts": error.attempts}
     except LLMError as error:
-        return {"status": "ERROR", "code": safe_code(error.code), "http_status": error.http_status}
+        result = {"status": "ERROR", "code": safe_code(error.code), "http_status": error.http_status}
+        if error.metadata is not None:
+            result['metadata'] = error.metadata
+        if error.diagnostics:
+            result['diagnostics'] = error.diagnostics
+        return result
 
 
 def main(argv=None):

src/viano_bug_analysis/llm/diagnostics.py

@@ -0,0 +1,43 @@
+"""Allowlisted failure details: never response text, headers or provider errors."""
+REASONS = {'stop','length','content_filter','tool_calls','function_call','MAX_TOKENS','SAFETY','STOP','OTHER','unknown','missing'}
+COUNTS = {'input_tokens','output_tokens','reasoning_tokens','response_bytes','output_characters','max_output_tokens'}
+KEYWORDS = {'type','required','additionalProperties','enum','const','pattern','minItems','maxItems','minimum','maximum','anyOf','oneOf'}
+
+
+def count(value):
+    return value if type(value) is int and 0 <= value <= 10**12 else None
+
+
+def sanitize(data):
+    if not isinstance(data,dict):
+        return {}
+    result = {k:count(v) for k,v in data.items() if k in COUNTS}
+    if 'finish_reason' in data:
+        value = data['finish_reason']
+        result['finish_reason'] = value if isinstance(value,str) and value in REASONS else ('missing' if value is None else 'unknown')
+    for key in ('has_refusal','has_tool_calls'):
+        if type(data.get(key)) is bool:
+            result[key] = data[key]
+    if isinstance(data.get('validation_keyword'),str) and data['validation_keyword'] in KEYWORDS:
+        result['validation_keyword'] = data['validation_keyword']
+    return result
+
+
+def envelope_details(envelope, max_output_tokens, response_bytes):
+    result = {'max_output_tokens':max_output_tokens,'response_bytes':response_bytes}
+    if isinstance(envelope,dict):
+        usage = envelope.get('usage')
+        if isinstance(usage,dict):
+            result.update(input_tokens=usage.get('prompt_tokens'),output_tokens=usage.get('completion_tokens'))
+            details = usage.get('completion_tokens_details')
+            if isinstance(details,dict):
+                result['reasoning_tokens'] = details.get('reasoning_tokens')
+        choices = envelope.get('choices')
+        if isinstance(choices,list) and choices and isinstance(choices[0],dict):
+            result['finish_reason'] = choices[0].get('finish_reason')
+            message = choices[0].get('message')
+            if isinstance(message,dict):
+                result.update(has_refusal=bool(message.get('refusal')),has_tool_calls=bool(message.get('tool_calls')))
+                if isinstance(message.get('content'),str):
+                    result['output_characters'] = len(message['content'])
+    return sanitize(result)

src/viano_bug_analysis/llm/errors.py

@@ -1,9 +1,12 @@
 class LLMError(Exception):
     """Safe diagnostic code; provider bodies and secrets are never included."""
 
-    def __init__(self, code: str, *, http_status=None):
+    def __init__(self, code: str, *, http_status=None, diagnostics=None):
         self.code = code
         self.http_status = http_status if type(http_status) is int and 100 <= http_status <= 599 else None
+        from .diagnostics import sanitize
+        self.diagnostics = sanitize(diagnostics)
+        self.metadata = None
         super().__init__(code)
 
 

src/viano_bug_analysis/llm/gateway.py

@@ -2,6 +2,7 @@ import asyncio
 import copy
 import logging
 import time
+import math
 
 from jsonschema import Draft202012Validator
 from jsonschema.exceptions import SchemaError, ValidationError
@@ -11,6 +12,8 @@ from viano_bug_analysis.package.manifest import FORMAT_CHECKER, parse_manifest
 
 from .errors import GatewayUnavailable, LLMError, ProviderUnavailable
 from .openai_compatible import OpenAICompatibleProvider
+from .provider_base import ProviderResponse
+from .diagnostics import sanitize, count
 
 LOGGER = logging.getLogger("viano_bug_analysis.llm")
 PRIVACY = {"local_only": {"local"}, "self_hosted": {"local", "self_hosted"}, "cloud_allowed": {"local", "self_hosted", "cloud"}}
@@ -71,6 +74,7 @@ class LLMGateway:
             if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]):
                 continue
             started = time.monotonic()
+            response = None
             try:
                 response = await asyncio.wait_for(self.adapters[provider.adapter].generate(provider, copy.deepcopy(request), max_response_bytes=self.config.max_response_bytes), timeout=provider.timeout_seconds)
                 if not isinstance(response.model, str) or not response.model.strip() or len(response.model) > 256 or any(ord(c) < 32 for c in response.model):
@@ -100,10 +104,34 @@ class LLMGateway:
                 if type(http_status) is int and 100 <= http_status <= 599:
                     attempts[-1]["http_status"] = http_status
                 LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code)
-            except ValidationError:
-                raise LLMError("LLM_OUTPUT_SCHEMA") from None
-            except LLMError:
+            except ValidationError as invalid:
+                error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics={'validation_keyword':invalid.validator})
+                self._failure_metadata(error,provider,request,started,response,attempts)
+                raise error from None
+            except LLMError as error:
+                self._failure_metadata(error,provider,request,started,response,attempts)
                 raise
             except Exception:
-                raise LLMError("LLM_PROVIDER_ERROR") from None
+                error = LLMError("LLM_PROVIDER_ERROR")
+                self._failure_metadata(error,provider,request,started,response,attempts)
+                raise error from None
         raise GatewayUnavailable(attempts)
+
+    @staticmethod
+    def _failure_metadata(error,provider,request,started,response,attempts):
+        details = sanitize(error.diagnostics)
+        if isinstance(response,ProviderResponse):
+            details = sanitize(response.diagnostics) | details
+            details.update(input_tokens=count(response.input_tokens),output_tokens=count(response.output_tokens))
+        details['max_output_tokens'] = count(provider.max_output_tokens)
+        error.diagnostics = details
+        input_tokens, output_tokens = details.get('input_tokens'),details.get('output_tokens')
+        cost = None
+        if all(v is not None for v in (input_tokens,output_tokens,provider.input_cost_per_million,provider.output_cost_per_million)):
+            value = (input_tokens*provider.input_cost_per_million+output_tokens*provider.output_cost_per_million)/1_000_000
+            if math.isfinite(value):
+                cost = value
+        error.metadata = {'provider':provider.id,'model':provider.model,'prompt_version':request['prompt_version'],
+                          'input_tokens':input_tokens,'output_tokens':output_tokens,'estimated_cost':cost,
+                          'cost_currency':provider.cost_currency if cost is not None else None,
+                          'elapsed_seconds':round(time.monotonic()-started,6),'attempts':copy.deepcopy(attempts)}

src/viano_bug_analysis/llm/openai_compatible.py

@@ -10,6 +10,7 @@ from viano_bug_analysis.package.manifest import parse_manifest
 
 from .errors import LLMError, ProviderUnavailable
 from .provider_base import ProviderResponse
+from .diagnostics import envelope_details
 
 
 class _NoRedirect(urllib.request.HTTPRedirectHandler):
@@ -62,24 +63,26 @@ class OpenAICompatibleProvider:
             raise LLMError("LLM_HTTP_CONFIGURATION", http_status=error.code) from None
         except (urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException):
             raise ProviderUnavailable("LLM_CONNECTION") from None
+        diagnostics = {'response_bytes':len(raw),'max_output_tokens':config.max_output_tokens}
         try:
             envelope = parse_manifest(raw)
+            diagnostics = envelope_details(envelope,config.max_output_tokens,len(raw))
             choice = envelope["choices"][0]
             message = choice["message"]
             if choice.get("finish_reason") != "stop" or message.get("refusal") or message.get("tool_calls"):
-                raise LLMError("LLM_INCOMPLETE_OUTPUT")
+                raise LLMError("LLM_INCOMPLETE_OUTPUT",diagnostics=diagnostics)
             content = message["content"]
             if not isinstance(content, str):
-                raise LLMError("LLM_RESPONSE")
+                raise LLMError("LLM_RESPONSE",diagnostics=diagnostics)
             document = parse_manifest(content.encode("utf-8"))
             model = envelope.get("model", config.model)
             if not isinstance(model, str) or not model.strip() or len(model) > 256 or any(ord(c) < 32 for c in model):
-                raise LLMError("LLM_RESPONSE")
+                raise LLMError("LLM_RESPONSE",diagnostics=diagnostics)
             usage = envelope.get("usage") or {}
             input_tokens, output_tokens = usage.get("prompt_tokens"), usage.get("completion_tokens")
             for count in (input_tokens, output_tokens):
                 if count is not None and (isinstance(count, bool) or not isinstance(count, int) or count < 0):
-                    raise LLMError("LLM_RESPONSE")
-            return ProviderResponse(document, model, input_tokens, output_tokens)
+                    raise LLMError("LLM_RESPONSE",diagnostics=diagnostics)
+            return ProviderResponse(document, model, input_tokens, output_tokens, diagnostics)
         except (ValueError, TypeError, KeyError, IndexError, AttributeError, RecursionError):
-            raise LLMError("LLM_RESPONSE") from None
+            raise LLMError("LLM_RESPONSE",diagnostics=diagnostics) from None

src/viano_bug_analysis/llm/provider_base.py

@@ -10,6 +10,7 @@ class ProviderResponse:
     model: str
     input_tokens: int | None = None
     output_tokens: int | None = None
+    diagnostics: dict | None = None
 
 
 class LLMProvider(Protocol):

src/viano_bug_analysis/llm/runner.py

@@ -152,6 +152,10 @@ class CaseLLMRunner:
             except LLMError as error:
                 details.update(status="ERROR", code=error.code)
                 outcome = {"code": error.code}
+                if error.metadata is not None:
+                    outcome['metadata'] = error.metadata
+                if error.diagnostics:
+                    outcome['diagnostics'] = error.diagnostics
                 if error.http_status is not None:
                     outcome["http_status"] = error.http_status
             result_ref = base + f"/round-{rounds:04d}.json"

src/viano_bug_analysis/schemas/llm-call-v1.schema.json

@@ -85,18 +85,133 @@
     "outcome": {
       "type": "object",
       "properties": {
-        "http_status": {"type": "integer", "minimum": 100, "maximum": 599},
-        "attempts": {"type": "array", "items": {"$ref": "#/$defs/http_attempt"}},
-        "metadata": {"type": "object", "properties": {
-          "attempts": {"type": "array", "items": {"$ref": "#/$defs/http_attempt"}}
-        }}
+        "http_status": {
+          "type": "integer",
+          "minimum": 100,
+          "maximum": 599
+        },
+        "attempts": {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/http_attempt"
+          }
+        },
+        "metadata": {
+          "type": "object",
+          "properties": {
+            "attempts": {
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/http_attempt"
+              }
+            }
+          }
+        },
+        "diagnostics": {
+          "type": "object",
+          "properties": {
+            "input_tokens": {
+              "type": [
+                "integer",
+                "null"
+              ],
+              "minimum": 0,
+              "maximum": 1000000000000
+            },
+            "max_output_tokens": {
+              "type": [
+                "integer",
+                "null"
+              ],
+              "minimum": 0,
+              "maximum": 1000000000000
+            },
+            "output_characters": {
+              "type": [
+                "integer",
+                "null"
+              ],
+              "minimum": 0,
+              "maximum": 1000000000000
+            },
+            "output_tokens": {
+              "type": [
+                "integer",
+                "null"
+              ],
+              "minimum": 0,
+              "maximum": 1000000000000
+            },
+            "reasoning_tokens": {
+              "type": [
+                "integer",
+                "null"
+              ],
+              "minimum": 0,
+              "maximum": 1000000000000
+            },
+            "response_bytes": {
+              "type": [
+                "integer",
+                "null"
+              ],
+              "minimum": 0,
+              "maximum": 1000000000000
+            },
+            "finish_reason": {
+              "enum": [
+                "MAX_TOKENS",
+                "OTHER",
+                "SAFETY",
+                "STOP",
+                "content_filter",
+                "function_call",
+                "length",
+                "missing",
+                "stop",
+                "tool_calls",
+                "unknown"
+              ]
+            },
+            "has_refusal": {
+              "type": "boolean"
+            },
+            "has_tool_calls": {
+              "type": "boolean"
+            },
+            "validation_keyword": {
+              "enum": [
+                "additionalProperties",
+                "anyOf",
+                "const",
+                "enum",
+                "maxItems",
+                "maximum",
+                "minItems",
+                "minimum",
+                "oneOf",
+                "pattern",
+                "required",
+                "type"
+              ]
+            }
+          },
+          "additionalProperties": false
+        }
       }
     }
   },
   "$defs": {
-    "http_attempt": {"type": "object", "properties": {
-      "http_status": {"type": "integer", "minimum": 100, "maximum": 599}
-    }}
+    "http_attempt": {
+      "type": "object",
+      "properties": {
+        "http_status": {
+          "type": "integer",
+          "minimum": 100,
+          "maximum": 599
+        }
+      }
+    }
   },
   "required": [
     "operation",

src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json

@@ -0,0 +1,92 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "input_tokens": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 0,
+      "maximum": 1000000000000
+    },
+    "max_output_tokens": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 0,
+      "maximum": 1000000000000
+    },
+    "output_characters": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 0,
+      "maximum": 1000000000000
+    },
+    "output_tokens": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 0,
+      "maximum": 1000000000000
+    },
+    "reasoning_tokens": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 0,
+      "maximum": 1000000000000
+    },
+    "response_bytes": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 0,
+      "maximum": 1000000000000
+    },
+    "finish_reason": {
+      "enum": [
+        "MAX_TOKENS",
+        "OTHER",
+        "SAFETY",
+        "STOP",
+        "content_filter",
+        "function_call",
+        "length",
+        "missing",
+        "stop",
+        "tool_calls",
+        "unknown"
+      ]
+    },
+    "has_refusal": {
+      "type": "boolean"
+    },
+    "has_tool_calls": {
+      "type": "boolean"
+    },
+    "validation_keyword": {
+      "enum": [
+        "additionalProperties",
+        "anyOf",
+        "const",
+        "enum",
+        "maxItems",
+        "maximum",
+        "minItems",
+        "minimum",
+        "oneOf",
+        "pattern",
+        "required",
+        "type"
+      ]
+    }
+  },
+  "additionalProperties": false
+}

src/viano_bug_analysis/telemetry/codes.py

@@ -19,3 +19,5 @@ KNOWN_CODES = KNOWN_CODES | frozenset({"INTERPRETATION_ENTITY_PROVENANCE"})
 
 KNOWN_CODES = KNOWN_CODES | frozenset({"INTERPRETATION_DESCRIPTION_SOURCES", "INTERPRETATION_PROVISIONAL_GAP_SOURCES"})
 KNOWN_MODULES = KNOWN_MODULES | frozenset({"viano_bug_analysis.analysis.quality"})
+
+KNOWN_MODULES = KNOWN_MODULES | frozenset({"viano_bug_analysis.llm.diagnostics"})

src/viano_bug_analysis/telemetry/core.py

@@ -368,6 +368,8 @@ class Telemetry:
             self.emit("llm_round", level="INFO" if status == "SUCCESS" else "WARNING", status=status, code=record.get("code"),
                       extra={"phase": phase, "provider_key": identity(metadata.get("provider")), "model_key": identity(metadata.get("model")),
                              "http_status": record["outcome"].get("http_status"),
+                               "diagnostics": record["outcome"].get("diagnostics"),
+                               "elapsed_seconds": metadata.get("elapsed_seconds"),
                              "input_tokens": metadata.get("input_tokens"), "output_tokens": metadata.get("output_tokens"),
                              "estimated_cost": cost if currency_valid else None,
                              "currency": currency if currency_valid else None,

src/viano_bug_analysis/telemetry/safety.py

@@ -96,13 +96,20 @@ def detail(event, value):
                 "package_generation": number(value.get("package_generation")), "analysis_revision": number(value.get("analysis_revision")),
                 "reason": safe_code(value.get("reason")) if value.get("reason") else None}
     if event == "llm_round":
-        return {"phase": value.get("phase") if value.get("phase") in STATES else None,
+        result = {"phase": value.get("phase") if value.get("phase") in STATES else None,
                 "http_status": http_status(value.get("http_status")),
                 "provider_key": hashed(value.get("provider_key")), "model_key": hashed(value.get("model_key")),
                 "input_tokens": number(value.get("input_tokens")), "output_tokens": number(value.get("output_tokens")),
                 "estimated_cost": number(value.get("estimated_cost")),
                 "currency": value.get("currency") if isinstance(value.get("currency"), str) and re.fullmatch("[A-Z]{3}", value["currency"]) else None,
                 "failed_providers": [{"provider_key": hashed(a.get("provider_key")), "code": safe_code(a.get("code")), "http_status": http_status(a.get("http_status"))} for a in value.get("failed_providers", [])[:64] if isinstance(a, dict)]}
+        from viano_bug_analysis.llm.diagnostics import sanitize
+        diagnostics = sanitize(value.get('diagnostics'))
+        if diagnostics:
+            result['diagnostics'] = diagnostics
+        if value.get('elapsed_seconds') is not None:
+            result['elapsed_seconds'] = number(value['elapsed_seconds'])
+        return result
     if event in {"operation_failed", "phase_diagnostic"}:
         chains = []
         for error in value.get("exceptions", [])[:8]:

tests/test_llm_failure_diagnostics.py

@@ -0,0 +1,105 @@
+import asyncio
+import json
+
+import pytest
+
+from test_llm_gateway import http_endpoint, active_case, request, provider, Adapter
+from viano_bug_analysis.llm import LLMError, LLMGateway, GatewayConfig, ProviderConfig, ProviderResponse, CaseLLMRunner
+from viano_bug_analysis.llm.diagnostics import sanitize
+from viano_bug_analysis.package import PackageReader, PackageValidator
+from viano_bug_analysis.telemetry.safety import detail
+
+
+@pytest.mark.parametrize('reason,refusal,tools', [('length',None,None),('content_filter','PRIVATE_REFUSAL',None),
+                                               ('tool_calls',None,[{'PRIVATE_TOOL':'PRIVATE_ARGUMENT'}])])
+def test_incomplete_api_responses_keep_usage_and_safe_finish_details(http_endpoint,reason,refusal,tools):
+    url,seen,reply=http_endpoint
+    reply['body']={'choices':[{'finish_reason':reason,'message':{'content':'PRIVATE_OUTPUT','refusal':refusal,'tool_calls':tools}}],
+                   'usage':{'prompt_tokens':123,'completion_tokens':8192,'completion_tokens_details':{'reasoning_tokens':8000}}}
+    config=GatewayConfig((ProviderConfig(id='test',model='configured-model',base_url=url,max_output_tokens=8192),))
+    with pytest.raises(LLMError,match='LLM_INCOMPLETE_OUTPUT') as caught:
+        asyncio.run(LLMGateway(config).complete_round(request()))
+    error=caught.value
+    assert error.diagnostics['finish_reason']==reason
+    assert error.diagnostics['has_refusal']==bool(refusal) and error.diagnostics['has_tool_calls']==bool(tools)
+    assert error.diagnostics['reasoning_tokens']==8000
+    assert error.metadata['input_tokens']==123 and error.metadata['output_tokens']==8192
+    assert error.metadata['elapsed_seconds']>=0
+    assert error.diagnostics['max_output_tokens']==8192
+    assert 'PRIVATE' not in json.dumps(error.diagnostics) and 'PRIVATE' not in str(error)
+    assert len(seen)==1
+
+
+def test_unknown_finish_reasons_and_bad_usage_are_not_copied(http_endpoint):
+    url,_,reply=http_endpoint
+    reply['body']={'choices':[{'finish_reason':'PRIVATE_REASON','message':{'content':'PRIVATE_BODY'}}],
+                   'usage':{'prompt_tokens':True,'completion_tokens':'PRIVATE_TOKEN_VALUE'}}
+    config=GatewayConfig((ProviderConfig(id='test',model='configured',base_url=url),))
+    with pytest.raises(LLMError) as caught:asyncio.run(LLMGateway(config).complete_round(request()))
+    error=caught.value
+    assert error.diagnostics['finish_reason']=='unknown'
+    assert error.metadata['input_tokens'] is None and error.metadata['output_tokens'] is None
+    assert 'PRIVATE' not in json.dumps({'metadata':error.metadata,'diagnostics':error.diagnostics})
+
+
+def test_failed_json_parsing_still_keeps_api_usage(http_endpoint):
+    url,_,reply=http_endpoint
+    reply['body']['choices'][0]['message']['content']='PRIVATE_NON_JSON'
+    config=GatewayConfig((ProviderConfig(id='test',model='configured',base_url=url),))
+    with pytest.raises(LLMError,match='LLM_RESPONSE') as caught:asyncio.run(LLMGateway(config).complete_round(request()))
+    assert caught.value.metadata['input_tokens']==3 and caught.value.diagnostics['finish_reason']=='stop'
+
+
+def test_http_schema_rejection_retains_completion_details(http_endpoint):
+    url,_,reply=http_endpoint
+    reply['body']['choices'][0]['message']['content']='{"wrong":1}'
+    config=GatewayConfig((ProviderConfig(id='test',model='configured',base_url=url),))
+    with pytest.raises(LLMError,match='LLM_OUTPUT_SCHEMA') as caught:asyncio.run(LLMGateway(config).complete_round(request()))
+    assert caught.value.diagnostics['finish_reason']=='stop'
+    assert caught.value.diagnostics['response_bytes']>0
+    assert caught.value.metadata['output_tokens']==5
+
+
+def test_schema_rejection_persists_usage_and_replays_without_network(active_case):
+    store,_,_=active_case
+    adapter=Adapter({'first':ProviderResponse({'PRIVATE_UNKNOWN_FIELD':'PRIVATE_VALUE'},'returned-model',100,20)})
+    config=GatewayConfig((provider(input_cost_per_million=2,output_cost_per_million=4),))
+    runner=CaseLLMRunner(store,LLMGateway(config,{'fixture':adapter}))
+    result=asyncio.run(runner.run('case-A','stable-package',operation_id='diagnostics',request=request()))
+    assert result.status=='ERROR' and PackageValidator().validate(result.package).valid
+    with PackageReader(result.package) as reader:
+        record=json.loads(reader.read_bytes(result.result_ref))
+    assert record['outcome']['metadata']['input_tokens']==100
+    assert record['outcome']['metadata']['estimated_cost']==pytest.approx(0.00028)
+    assert record['outcome']['diagnostics']['validation_keyword'] in {'required','additionalProperties'}
+    assert 'PRIVATE' not in json.dumps(record['outcome'])
+    assert asyncio.run(runner.run('case-A','stable-package',operation_id='diagnostics',request=request()))==result
+    assert adapter.calls==['first']
+
+
+def test_safe_diagnostics_survive_log_sanitization():
+    dirty={'finish_reason':'length','input_tokens':3,'output_tokens':8,'has_refusal':True,
+           'provider_body':'PRIVATE_BODY','validation_keyword':{},'output_characters':4}
+    cleaned=sanitize(dirty)
+    assert 'provider_body' not in cleaned and 'validation_keyword' not in cleaned
+    event=detail('llm_round',{'diagnostics':dirty,'elapsed_seconds':1.25})
+    assert event['diagnostics']==cleaned and event['elapsed_seconds']==1.25
+    assert 'PRIVATE' not in json.dumps(event)
+
+
+def test_failed_usage_is_counted_and_finish_reason_visible_in_service_log(active_case):
+    from viano_bug_analysis.telemetry import Telemetry
+    from viano_bug_analysis.telemetry.__main__ import read_events
+    store,_,_=active_case
+    error=LLMError('LLM_INCOMPLETE_OUTPUT',diagnostics={'finish_reason':'length','input_tokens':10,'output_tokens':7})
+    adapter=Adapter({'first':error})
+    with Telemetry(store.workspace) as observer:
+        store.telemetry=observer
+        runner=CaseLLMRunner(store,LLMGateway(GatewayConfig((provider(),)),{'fixture':adapter}))
+        result=asyncio.run(runner.run('case-A','stable-package',operation_id='incomplete',request=request()))
+        assert result.status=='ERROR'
+    event=next(e for e in read_events(store.workspace.root) if e['event']=='llm_round')
+    assert event['detail']['diagnostics']['finish_reason']=='length'
+    metrics=(observer.directory/'metrics.prom').read_text()
+    assert 'vba_llm_tokens_total{direction="input_tokens"} 10' in metrics
+    assert 'vba_llm_tokens_total{direction="output_tokens"} 7' in metrics