diff --git a/docs/llm-failure-diagnostics-v1.md b/docs/llm-failure-diagnostics-v1.md new file mode 100644 index 0000000..d06d187 --- /dev/null +++ b/docs/llm-failure-diagnostics-v1.md @@ -0,0 +1,44 @@ +# LLM failure diagnostics + +Failed response processing now retains optional `outcome.metadata` and +`outcome.diagnostics` in the existing v1 `llm/operations/.../round-NNNN.json` +records. Old records without these fields remain valid and replay without API +calls. Previously discarded details cannot be recovered retrospectively. + +For incomplete replies, diagnostics preserve: + +- The API finish reason, restricted to known values; absent/unrecognized values + are labeled `missing`/`unknown` rather than copying arbitrary provider text. +- Boolean refusal/tool-call indicators, not their text or arguments. +- Input/output tokens and reasoning tokens when supplied by the API. +- Configured output-token limit, received response bytes and output character count. + +Metadata records the configured provider/model, prompt version, elapsed seconds, +known usage, previous unavailable-provider attempts, and estimated cost only when +usage and operator-configured prices are available. Missing/invalid token values +remain unknown, not zero. API output token usage may include reasoning, depending +on the provider; do not infer reasoning usage when the API omits that field. + +JSON parsing errors retain available envelope details. Schema rejection retains +usage and a safe validation keyword, without storing response contents or the +validation exception message. Diagnostic fields have their own allowlisted JSON +schema (`llm-failure-diagnostics-v1.schema.json`). No headers, API keys, provider +error bodies, refusal text, tool arguments or partial completion text are added. + +The same safe details are visible in structured service `llm_round` log events. +Known usage/cost from failed replies now contributes to existing telemetry +counters. The connection-test CLI includes diagnostic/metadata fields on errors. +Existing retry, failover, validation limits, lifecycle and hypothesis budgets are +unchanged. Successful reply metadata and response contracts remain unchanged. + +A `finish_reason = length` indicates an output-budget stop reported by the API; +compare the output token count with `max_output_tokens`, and inspect reasoning +usage if available. `content_filter`, refusal/tool-call indicators, or missing +finish reasons describe different failure conditions. An incomplete response +alone does not prove a transport timeout. Transport/availability failures still +use their existing codes and HTTP attempt records; usage may be unavailable. + +Tests use local HTTP fixtures and model fixtures. They cover truncation, +refusal/tool calls, malformed usage, unknown finish reasons, JSON/schema failures, +usage persistence, replay without network calls, telemetry counters/logs and +removal of free provider text. No paid model request is needed. diff --git a/src/viano_bug_analysis/connection_test.py b/src/viano_bug_analysis/connection_test.py index 8703f85..0f0f1c5 100644 --- a/src/viano_bug_analysis/connection_test.py +++ b/src/viano_bug_analysis/connection_test.py @@ -21,7 +21,12 @@ async def probe(gateway): except GatewayUnavailable as error: return {"status": "UNAVAILABLE", "attempts": error.attempts} except LLMError as error: - return {"status": "ERROR", "code": safe_code(error.code), "http_status": error.http_status} + result = {"status": "ERROR", "code": safe_code(error.code), "http_status": error.http_status} + if error.metadata is not None: + result['metadata'] = error.metadata + if error.diagnostics: + result['diagnostics'] = error.diagnostics + return result def main(argv=None): diff --git a/src/viano_bug_analysis/llm/diagnostics.py b/src/viano_bug_analysis/llm/diagnostics.py new file mode 100644 index 0000000..d445654 --- /dev/null +++ b/src/viano_bug_analysis/llm/diagnostics.py @@ -0,0 +1,43 @@ +"""Allowlisted failure details: never response text, headers or provider errors.""" +REASONS = {'stop','length','content_filter','tool_calls','function_call','MAX_TOKENS','SAFETY','STOP','OTHER','unknown','missing'} +COUNTS = {'input_tokens','output_tokens','reasoning_tokens','response_bytes','output_characters','max_output_tokens'} +KEYWORDS = {'type','required','additionalProperties','enum','const','pattern','minItems','maxItems','minimum','maximum','anyOf','oneOf'} + + +def count(value): + return value if type(value) is int and 0 <= value <= 10**12 else None + + +def sanitize(data): + if not isinstance(data,dict): + return {} + result = {k:count(v) for k,v in data.items() if k in COUNTS} + if 'finish_reason' in data: + value = data['finish_reason'] + result['finish_reason'] = value if isinstance(value,str) and value in REASONS else ('missing' if value is None else 'unknown') + for key in ('has_refusal','has_tool_calls'): + if type(data.get(key)) is bool: + result[key] = data[key] + if isinstance(data.get('validation_keyword'),str) and data['validation_keyword'] in KEYWORDS: + result['validation_keyword'] = data['validation_keyword'] + return result + + +def envelope_details(envelope, max_output_tokens, response_bytes): + result = {'max_output_tokens':max_output_tokens,'response_bytes':response_bytes} + if isinstance(envelope,dict): + usage = envelope.get('usage') + if isinstance(usage,dict): + result.update(input_tokens=usage.get('prompt_tokens'),output_tokens=usage.get('completion_tokens')) + details = usage.get('completion_tokens_details') + if isinstance(details,dict): + result['reasoning_tokens'] = details.get('reasoning_tokens') + choices = envelope.get('choices') + if isinstance(choices,list) and choices and isinstance(choices[0],dict): + result['finish_reason'] = choices[0].get('finish_reason') + message = choices[0].get('message') + if isinstance(message,dict): + result.update(has_refusal=bool(message.get('refusal')),has_tool_calls=bool(message.get('tool_calls'))) + if isinstance(message.get('content'),str): + result['output_characters'] = len(message['content']) + return sanitize(result) diff --git a/src/viano_bug_analysis/llm/errors.py b/src/viano_bug_analysis/llm/errors.py index c78e773..221f41e 100644 --- a/src/viano_bug_analysis/llm/errors.py +++ b/src/viano_bug_analysis/llm/errors.py @@ -1,9 +1,12 @@ class LLMError(Exception): """Safe diagnostic code; provider bodies and secrets are never included.""" - def __init__(self, code: str, *, http_status=None): + def __init__(self, code: str, *, http_status=None, diagnostics=None): self.code = code self.http_status = http_status if type(http_status) is int and 100 <= http_status <= 599 else None + from .diagnostics import sanitize + self.diagnostics = sanitize(diagnostics) + self.metadata = None super().__init__(code) diff --git a/src/viano_bug_analysis/llm/gateway.py b/src/viano_bug_analysis/llm/gateway.py index 0dac679..9f388e8 100644 --- a/src/viano_bug_analysis/llm/gateway.py +++ b/src/viano_bug_analysis/llm/gateway.py @@ -2,6 +2,7 @@ import asyncio import copy import logging import time +import math from jsonschema import Draft202012Validator from jsonschema.exceptions import SchemaError, ValidationError @@ -11,6 +12,8 @@ from viano_bug_analysis.package.manifest import FORMAT_CHECKER, parse_manifest from .errors import GatewayUnavailable, LLMError, ProviderUnavailable from .openai_compatible import OpenAICompatibleProvider +from .provider_base import ProviderResponse +from .diagnostics import sanitize, count LOGGER = logging.getLogger("viano_bug_analysis.llm") PRIVACY = {"local_only": {"local"}, "self_hosted": {"local", "self_hosted"}, "cloud_allowed": {"local", "self_hosted", "cloud"}} @@ -71,6 +74,7 @@ class LLMGateway: if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]): continue started = time.monotonic() + response = None try: response = await asyncio.wait_for(self.adapters[provider.adapter].generate(provider, copy.deepcopy(request), max_response_bytes=self.config.max_response_bytes), timeout=provider.timeout_seconds) if not isinstance(response.model, str) or not response.model.strip() or len(response.model) > 256 or any(ord(c) < 32 for c in response.model): @@ -100,10 +104,34 @@ class LLMGateway: if type(http_status) is int and 100 <= http_status <= 599: attempts[-1]["http_status"] = http_status LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code) - except ValidationError: - raise LLMError("LLM_OUTPUT_SCHEMA") from None - except LLMError: + except ValidationError as invalid: + error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics={'validation_keyword':invalid.validator}) + self._failure_metadata(error,provider,request,started,response,attempts) + raise error from None + except LLMError as error: + self._failure_metadata(error,provider,request,started,response,attempts) raise except Exception: - raise LLMError("LLM_PROVIDER_ERROR") from None + error = LLMError("LLM_PROVIDER_ERROR") + self._failure_metadata(error,provider,request,started,response,attempts) + raise error from None raise GatewayUnavailable(attempts) + + @staticmethod + def _failure_metadata(error,provider,request,started,response,attempts): + details = sanitize(error.diagnostics) + if isinstance(response,ProviderResponse): + details = sanitize(response.diagnostics) | details + details.update(input_tokens=count(response.input_tokens),output_tokens=count(response.output_tokens)) + details['max_output_tokens'] = count(provider.max_output_tokens) + error.diagnostics = details + input_tokens, output_tokens = details.get('input_tokens'),details.get('output_tokens') + cost = None + if all(v is not None for v in (input_tokens,output_tokens,provider.input_cost_per_million,provider.output_cost_per_million)): + value = (input_tokens*provider.input_cost_per_million+output_tokens*provider.output_cost_per_million)/1_000_000 + if math.isfinite(value): + cost = value + error.metadata = {'provider':provider.id,'model':provider.model,'prompt_version':request['prompt_version'], + 'input_tokens':input_tokens,'output_tokens':output_tokens,'estimated_cost':cost, + 'cost_currency':provider.cost_currency if cost is not None else None, + 'elapsed_seconds':round(time.monotonic()-started,6),'attempts':copy.deepcopy(attempts)} diff --git a/src/viano_bug_analysis/llm/openai_compatible.py b/src/viano_bug_analysis/llm/openai_compatible.py index 93cd6af..e87612a 100644 --- a/src/viano_bug_analysis/llm/openai_compatible.py +++ b/src/viano_bug_analysis/llm/openai_compatible.py @@ -10,6 +10,7 @@ from viano_bug_analysis.package.manifest import parse_manifest from .errors import LLMError, ProviderUnavailable from .provider_base import ProviderResponse +from .diagnostics import envelope_details class _NoRedirect(urllib.request.HTTPRedirectHandler): @@ -62,24 +63,26 @@ class OpenAICompatibleProvider: raise LLMError("LLM_HTTP_CONFIGURATION", http_status=error.code) from None except (urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException): raise ProviderUnavailable("LLM_CONNECTION") from None + diagnostics = {'response_bytes':len(raw),'max_output_tokens':config.max_output_tokens} try: envelope = parse_manifest(raw) + diagnostics = envelope_details(envelope,config.max_output_tokens,len(raw)) choice = envelope["choices"][0] message = choice["message"] if choice.get("finish_reason") != "stop" or message.get("refusal") or message.get("tool_calls"): - raise LLMError("LLM_INCOMPLETE_OUTPUT") + raise LLMError("LLM_INCOMPLETE_OUTPUT",diagnostics=diagnostics) content = message["content"] if not isinstance(content, str): - raise LLMError("LLM_RESPONSE") + raise LLMError("LLM_RESPONSE",diagnostics=diagnostics) document = parse_manifest(content.encode("utf-8")) model = envelope.get("model", config.model) if not isinstance(model, str) or not model.strip() or len(model) > 256 or any(ord(c) < 32 for c in model): - raise LLMError("LLM_RESPONSE") + raise LLMError("LLM_RESPONSE",diagnostics=diagnostics) usage = envelope.get("usage") or {} input_tokens, output_tokens = usage.get("prompt_tokens"), usage.get("completion_tokens") for count in (input_tokens, output_tokens): if count is not None and (isinstance(count, bool) or not isinstance(count, int) or count < 0): - raise LLMError("LLM_RESPONSE") - return ProviderResponse(document, model, input_tokens, output_tokens) + raise LLMError("LLM_RESPONSE",diagnostics=diagnostics) + return ProviderResponse(document, model, input_tokens, output_tokens, diagnostics) except (ValueError, TypeError, KeyError, IndexError, AttributeError, RecursionError): - raise LLMError("LLM_RESPONSE") from None + raise LLMError("LLM_RESPONSE",diagnostics=diagnostics) from None diff --git a/src/viano_bug_analysis/llm/provider_base.py b/src/viano_bug_analysis/llm/provider_base.py index 0af3ed5..735c3ce 100644 --- a/src/viano_bug_analysis/llm/provider_base.py +++ b/src/viano_bug_analysis/llm/provider_base.py @@ -10,6 +10,7 @@ class ProviderResponse: model: str input_tokens: int | None = None output_tokens: int | None = None + diagnostics: dict | None = None class LLMProvider(Protocol): diff --git a/src/viano_bug_analysis/llm/runner.py b/src/viano_bug_analysis/llm/runner.py index a055068..422431b 100644 --- a/src/viano_bug_analysis/llm/runner.py +++ b/src/viano_bug_analysis/llm/runner.py @@ -152,6 +152,10 @@ class CaseLLMRunner: except LLMError as error: details.update(status="ERROR", code=error.code) outcome = {"code": error.code} + if error.metadata is not None: + outcome['metadata'] = error.metadata + if error.diagnostics: + outcome['diagnostics'] = error.diagnostics if error.http_status is not None: outcome["http_status"] = error.http_status result_ref = base + f"/round-{rounds:04d}.json" diff --git a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json index 91bd83f..f639d43 100644 --- a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json @@ -85,18 +85,133 @@ "outcome": { "type": "object", "properties": { - "http_status": {"type": "integer", "minimum": 100, "maximum": 599}, - "attempts": {"type": "array", "items": {"$ref": "#/$defs/http_attempt"}}, - "metadata": {"type": "object", "properties": { - "attempts": {"type": "array", "items": {"$ref": "#/$defs/http_attempt"}} - }} + "http_status": { + "type": "integer", + "minimum": 100, + "maximum": 599 + }, + "attempts": { + "type": "array", + "items": { + "$ref": "#/$defs/http_attempt" + } + }, + "metadata": { + "type": "object", + "properties": { + "attempts": { + "type": "array", + "items": { + "$ref": "#/$defs/http_attempt" + } + } + } + }, + "diagnostics": { + "type": "object", + "properties": { + "input_tokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "max_output_tokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "output_characters": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "output_tokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "reasoning_tokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "response_bytes": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "finish_reason": { + "enum": [ + "MAX_TOKENS", + "OTHER", + "SAFETY", + "STOP", + "content_filter", + "function_call", + "length", + "missing", + "stop", + "tool_calls", + "unknown" + ] + }, + "has_refusal": { + "type": "boolean" + }, + "has_tool_calls": { + "type": "boolean" + }, + "validation_keyword": { + "enum": [ + "additionalProperties", + "anyOf", + "const", + "enum", + "maxItems", + "maximum", + "minItems", + "minimum", + "oneOf", + "pattern", + "required", + "type" + ] + } + }, + "additionalProperties": false + } } } }, "$defs": { - "http_attempt": {"type": "object", "properties": { - "http_status": {"type": "integer", "minimum": 100, "maximum": 599} - }} + "http_attempt": { + "type": "object", + "properties": { + "http_status": { + "type": "integer", + "minimum": 100, + "maximum": 599 + } + } + } }, "required": [ "operation", diff --git a/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json b/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json new file mode 100644 index 0000000..4d36d72 --- /dev/null +++ b/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json @@ -0,0 +1,92 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "input_tokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "max_output_tokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "output_characters": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "output_tokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "reasoning_tokens": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "response_bytes": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 1000000000000 + }, + "finish_reason": { + "enum": [ + "MAX_TOKENS", + "OTHER", + "SAFETY", + "STOP", + "content_filter", + "function_call", + "length", + "missing", + "stop", + "tool_calls", + "unknown" + ] + }, + "has_refusal": { + "type": "boolean" + }, + "has_tool_calls": { + "type": "boolean" + }, + "validation_keyword": { + "enum": [ + "additionalProperties", + "anyOf", + "const", + "enum", + "maxItems", + "maximum", + "minItems", + "minimum", + "oneOf", + "pattern", + "required", + "type" + ] + } + }, + "additionalProperties": false +} diff --git a/src/viano_bug_analysis/telemetry/codes.py b/src/viano_bug_analysis/telemetry/codes.py index b6da33a..6ac9c0f 100644 --- a/src/viano_bug_analysis/telemetry/codes.py +++ b/src/viano_bug_analysis/telemetry/codes.py @@ -19,3 +19,5 @@ KNOWN_CODES = KNOWN_CODES | frozenset({"INTERPRETATION_ENTITY_PROVENANCE"}) KNOWN_CODES = KNOWN_CODES | frozenset({"INTERPRETATION_DESCRIPTION_SOURCES", "INTERPRETATION_PROVISIONAL_GAP_SOURCES"}) KNOWN_MODULES = KNOWN_MODULES | frozenset({"viano_bug_analysis.analysis.quality"}) + +KNOWN_MODULES = KNOWN_MODULES | frozenset({"viano_bug_analysis.llm.diagnostics"}) diff --git a/src/viano_bug_analysis/telemetry/core.py b/src/viano_bug_analysis/telemetry/core.py index 44cb00a..a6181da 100644 --- a/src/viano_bug_analysis/telemetry/core.py +++ b/src/viano_bug_analysis/telemetry/core.py @@ -368,6 +368,8 @@ class Telemetry: self.emit("llm_round", level="INFO" if status == "SUCCESS" else "WARNING", status=status, code=record.get("code"), extra={"phase": phase, "provider_key": identity(metadata.get("provider")), "model_key": identity(metadata.get("model")), "http_status": record["outcome"].get("http_status"), + "diagnostics": record["outcome"].get("diagnostics"), + "elapsed_seconds": metadata.get("elapsed_seconds"), "input_tokens": metadata.get("input_tokens"), "output_tokens": metadata.get("output_tokens"), "estimated_cost": cost if currency_valid else None, "currency": currency if currency_valid else None, diff --git a/src/viano_bug_analysis/telemetry/safety.py b/src/viano_bug_analysis/telemetry/safety.py index d6fa6db..b608e1a 100644 --- a/src/viano_bug_analysis/telemetry/safety.py +++ b/src/viano_bug_analysis/telemetry/safety.py @@ -96,13 +96,20 @@ def detail(event, value): "package_generation": number(value.get("package_generation")), "analysis_revision": number(value.get("analysis_revision")), "reason": safe_code(value.get("reason")) if value.get("reason") else None} if event == "llm_round": - return {"phase": value.get("phase") if value.get("phase") in STATES else None, + result = {"phase": value.get("phase") if value.get("phase") in STATES else None, "http_status": http_status(value.get("http_status")), "provider_key": hashed(value.get("provider_key")), "model_key": hashed(value.get("model_key")), "input_tokens": number(value.get("input_tokens")), "output_tokens": number(value.get("output_tokens")), "estimated_cost": number(value.get("estimated_cost")), "currency": value.get("currency") if isinstance(value.get("currency"), str) and re.fullmatch("[A-Z]{3}", value["currency"]) else None, "failed_providers": [{"provider_key": hashed(a.get("provider_key")), "code": safe_code(a.get("code")), "http_status": http_status(a.get("http_status"))} for a in value.get("failed_providers", [])[:64] if isinstance(a, dict)]} + from viano_bug_analysis.llm.diagnostics import sanitize + diagnostics = sanitize(value.get('diagnostics')) + if diagnostics: + result['diagnostics'] = diagnostics + if value.get('elapsed_seconds') is not None: + result['elapsed_seconds'] = number(value['elapsed_seconds']) + return result if event in {"operation_failed", "phase_diagnostic"}: chains = [] for error in value.get("exceptions", [])[:8]: diff --git a/tests/test_llm_failure_diagnostics.py b/tests/test_llm_failure_diagnostics.py new file mode 100644 index 0000000..2a832a1 --- /dev/null +++ b/tests/test_llm_failure_diagnostics.py @@ -0,0 +1,105 @@ +import asyncio +import json + +import pytest + +from test_llm_gateway import http_endpoint, active_case, request, provider, Adapter +from viano_bug_analysis.llm import LLMError, LLMGateway, GatewayConfig, ProviderConfig, ProviderResponse, CaseLLMRunner +from viano_bug_analysis.llm.diagnostics import sanitize +from viano_bug_analysis.package import PackageReader, PackageValidator +from viano_bug_analysis.telemetry.safety import detail + + +@pytest.mark.parametrize('reason,refusal,tools', [('length',None,None),('content_filter','PRIVATE_REFUSAL',None), + ('tool_calls',None,[{'PRIVATE_TOOL':'PRIVATE_ARGUMENT'}])]) +def test_incomplete_api_responses_keep_usage_and_safe_finish_details(http_endpoint,reason,refusal,tools): + url,seen,reply=http_endpoint + reply['body']={'choices':[{'finish_reason':reason,'message':{'content':'PRIVATE_OUTPUT','refusal':refusal,'tool_calls':tools}}], + 'usage':{'prompt_tokens':123,'completion_tokens':8192,'completion_tokens_details':{'reasoning_tokens':8000}}} + config=GatewayConfig((ProviderConfig(id='test',model='configured-model',base_url=url,max_output_tokens=8192),)) + with pytest.raises(LLMError,match='LLM_INCOMPLETE_OUTPUT') as caught: + asyncio.run(LLMGateway(config).complete_round(request())) + error=caught.value + assert error.diagnostics['finish_reason']==reason + assert error.diagnostics['has_refusal']==bool(refusal) and error.diagnostics['has_tool_calls']==bool(tools) + assert error.diagnostics['reasoning_tokens']==8000 + assert error.metadata['input_tokens']==123 and error.metadata['output_tokens']==8192 + assert error.metadata['elapsed_seconds']>=0 + assert error.diagnostics['max_output_tokens']==8192 + assert 'PRIVATE' not in json.dumps(error.diagnostics) and 'PRIVATE' not in str(error) + assert len(seen)==1 + + +def test_unknown_finish_reasons_and_bad_usage_are_not_copied(http_endpoint): + url,_,reply=http_endpoint + reply['body']={'choices':[{'finish_reason':'PRIVATE_REASON','message':{'content':'PRIVATE_BODY'}}], + 'usage':{'prompt_tokens':True,'completion_tokens':'PRIVATE_TOKEN_VALUE'}} + config=GatewayConfig((ProviderConfig(id='test',model='configured',base_url=url),)) + with pytest.raises(LLMError) as caught:asyncio.run(LLMGateway(config).complete_round(request())) + error=caught.value + assert error.diagnostics['finish_reason']=='unknown' + assert error.metadata['input_tokens'] is None and error.metadata['output_tokens'] is None + assert 'PRIVATE' not in json.dumps({'metadata':error.metadata,'diagnostics':error.diagnostics}) + + +def test_failed_json_parsing_still_keeps_api_usage(http_endpoint): + url,_,reply=http_endpoint + reply['body']['choices'][0]['message']['content']='PRIVATE_NON_JSON' + config=GatewayConfig((ProviderConfig(id='test',model='configured',base_url=url),)) + with pytest.raises(LLMError,match='LLM_RESPONSE') as caught:asyncio.run(LLMGateway(config).complete_round(request())) + assert caught.value.metadata['input_tokens']==3 and caught.value.diagnostics['finish_reason']=='stop' + + +def test_http_schema_rejection_retains_completion_details(http_endpoint): + url,_,reply=http_endpoint + reply['body']['choices'][0]['message']['content']='{"wrong":1}' + config=GatewayConfig((ProviderConfig(id='test',model='configured',base_url=url),)) + with pytest.raises(LLMError,match='LLM_OUTPUT_SCHEMA') as caught:asyncio.run(LLMGateway(config).complete_round(request())) + assert caught.value.diagnostics['finish_reason']=='stop' + assert caught.value.diagnostics['response_bytes']>0 + assert caught.value.metadata['output_tokens']==5 + + +def test_schema_rejection_persists_usage_and_replays_without_network(active_case): + store,_,_=active_case + adapter=Adapter({'first':ProviderResponse({'PRIVATE_UNKNOWN_FIELD':'PRIVATE_VALUE'},'returned-model',100,20)}) + config=GatewayConfig((provider(input_cost_per_million=2,output_cost_per_million=4),)) + runner=CaseLLMRunner(store,LLMGateway(config,{'fixture':adapter})) + result=asyncio.run(runner.run('case-A','stable-package',operation_id='diagnostics',request=request())) + assert result.status=='ERROR' and PackageValidator().validate(result.package).valid + with PackageReader(result.package) as reader: + record=json.loads(reader.read_bytes(result.result_ref)) + assert record['outcome']['metadata']['input_tokens']==100 + assert record['outcome']['metadata']['estimated_cost']==pytest.approx(0.00028) + assert record['outcome']['diagnostics']['validation_keyword'] in {'required','additionalProperties'} + assert 'PRIVATE' not in json.dumps(record['outcome']) + assert asyncio.run(runner.run('case-A','stable-package',operation_id='diagnostics',request=request()))==result + assert adapter.calls==['first'] + + +def test_safe_diagnostics_survive_log_sanitization(): + dirty={'finish_reason':'length','input_tokens':3,'output_tokens':8,'has_refusal':True, + 'provider_body':'PRIVATE_BODY','validation_keyword':{},'output_characters':4} + cleaned=sanitize(dirty) + assert 'provider_body' not in cleaned and 'validation_keyword' not in cleaned + event=detail('llm_round',{'diagnostics':dirty,'elapsed_seconds':1.25}) + assert event['diagnostics']==cleaned and event['elapsed_seconds']==1.25 + assert 'PRIVATE' not in json.dumps(event) + + +def test_failed_usage_is_counted_and_finish_reason_visible_in_service_log(active_case): + from viano_bug_analysis.telemetry import Telemetry + from viano_bug_analysis.telemetry.__main__ import read_events + store,_,_=active_case + error=LLMError('LLM_INCOMPLETE_OUTPUT',diagnostics={'finish_reason':'length','input_tokens':10,'output_tokens':7}) + adapter=Adapter({'first':error}) + with Telemetry(store.workspace) as observer: + store.telemetry=observer + runner=CaseLLMRunner(store,LLMGateway(GatewayConfig((provider(),)),{'fixture':adapter})) + result=asyncio.run(runner.run('case-A','stable-package',operation_id='incomplete',request=request())) + assert result.status=='ERROR' + event=next(e for e in read_events(store.workspace.root) if e['event']=='llm_round') + assert event['detail']['diagnostics']['finish_reason']=='length' + metrics=(observer.directory/'metrics.prom').read_text() + assert 'vba_llm_tokens_total{direction="input_tokens"} 10' in metrics + assert 'vba_llm_tokens_total{direction="output_tokens"} 7' in metrics