From 4d0edc030d959fe488020b53618523ce49487cd1 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Wed, 14 Oct 2026 16:21:33 +0200 Subject: [PATCH] Preserve HTTP status codes in LLM history and safe telemetry --- docs/llm-gateway-v1.md | 7 +++ docs/telemetry-v1.md | 7 +++ src/viano_bug_analysis/llm/errors.py | 3 +- src/viano_bug_analysis/llm/gateway.py | 3 + .../llm/openai_compatible.py | 4 +- src/viano_bug_analysis/llm/runner.py | 2 + .../schemas/llm-call-v1.schema.json | 14 ++++- src/viano_bug_analysis/telemetry/core.py | 3 +- src/viano_bug_analysis/telemetry/safety.py | 7 ++- tests/test_llm_gateway.py | 56 ++++++++++++++++++- 10 files changed, 99 insertions(+), 7 deletions(-) diff --git a/docs/llm-gateway-v1.md b/docs/llm-gateway-v1.md index c1f4557..1a020b7 100644 --- a/docs/llm-gateway-v1.md +++ b/docs/llm-gateway-v1.md @@ -51,6 +51,13 @@ other HTTP configuration failures, refusals, truncated model output, invalid JSO schema violations are controlled errors; they are not logged as unreachable servers. Adapters execute one call and never retry independently. +HTTP failures additionally preserve an optional numeric `http_status` (100–599). +Temporary failures record it in `outcome.attempts` or successful failover metadata; +nonretryable HTTP errors record it in `outcome.http_status`. Existing v1 records +without this field remain valid. No provider response body, headers or credentials +are retained. Retry/failover classification is unchanged: 408/429/5xx are temporary; +a 429 alone does not distinguish rate limits from exhausted quota. + Operational defaults: | Setting | Default | diff --git a/docs/telemetry-v1.md b/docs/telemetry-v1.md index fa9d3e0..8ee5457 100644 --- a/docs/telemetry-v1.md +++ b/docs/telemetry-v1.md @@ -1,5 +1,12 @@ # Operational telemetry v1 +LLM round events expose numeric `http_status` for nonretryable HTTP failures and +for every entry in `failed_providers`. Use the `events` CLI to inspect temporary +provider failures (warning events). For example, 429 indicates a quota/rate-limit +response; 503 indicates service unavailability. Missing status on older records, +timeouts or connection errors means unknown, not success. Provider bodies and +headers are never logged. This field does not change retries or case budgets. + Telemetry observes existing library execution, not case truth. It never resumes BLOCKED cases, refills budgets, changes routing, retries a model, edits packages, executes tools or deletes case artifacts. `docs/architecture.md` is unchanged. diff --git a/src/viano_bug_analysis/llm/errors.py b/src/viano_bug_analysis/llm/errors.py index dfc6c73..c78e773 100644 --- a/src/viano_bug_analysis/llm/errors.py +++ b/src/viano_bug_analysis/llm/errors.py @@ -1,8 +1,9 @@ class LLMError(Exception): """Safe diagnostic code; provider bodies and secrets are never included.""" - def __init__(self, code: str): + def __init__(self, code: str, *, http_status=None): self.code = code + self.http_status = http_status if type(http_status) is int and 100 <= http_status <= 599 else None super().__init__(code) diff --git a/src/viano_bug_analysis/llm/gateway.py b/src/viano_bug_analysis/llm/gateway.py index 45c8962..0dac679 100644 --- a/src/viano_bug_analysis/llm/gateway.py +++ b/src/viano_bug_analysis/llm/gateway.py @@ -96,6 +96,9 @@ class LLMGateway: except (ProviderUnavailable, TimeoutError) as error: code = error.code if isinstance(error, ProviderUnavailable) else "LLM_TIMEOUT" attempts.append({"provider": provider.id, "code": code}) + http_status = getattr(error, "http_status", None) + if type(http_status) is int and 100 <= http_status <= 599: + attempts[-1]["http_status"] = http_status LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code) except ValidationError: raise LLMError("LLM_OUTPUT_SCHEMA") from None diff --git a/src/viano_bug_analysis/llm/openai_compatible.py b/src/viano_bug_analysis/llm/openai_compatible.py index 7e368f6..93cd6af 100644 --- a/src/viano_bug_analysis/llm/openai_compatible.py +++ b/src/viano_bug_analysis/llm/openai_compatible.py @@ -58,8 +58,8 @@ class OpenAICompatibleProvider: except urllib.error.HTTPError as error: error.close() if error.code in {408, 429} or 500 <= error.code < 600: - raise ProviderUnavailable("LLM_HTTP_TEMPORARY") from None - raise LLMError("LLM_HTTP_CONFIGURATION") from None + raise ProviderUnavailable("LLM_HTTP_TEMPORARY", http_status=error.code) from None + raise LLMError("LLM_HTTP_CONFIGURATION", http_status=error.code) from None except (urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException): raise ProviderUnavailable("LLM_CONNECTION") from None try: diff --git a/src/viano_bug_analysis/llm/runner.py b/src/viano_bug_analysis/llm/runner.py index 23b3f8c..a055068 100644 --- a/src/viano_bug_analysis/llm/runner.py +++ b/src/viano_bug_analysis/llm/runner.py @@ -152,6 +152,8 @@ class CaseLLMRunner: except LLMError as error: details.update(status="ERROR", code=error.code) outcome = {"code": error.code} + if error.http_status is not None: + outcome["http_status"] = error.http_status result_ref = base + f"/round-{rounds:04d}.json" details["result_ref"] = result_ref record = {"format": "viano-llm-call", "format_version": 1, diff --git a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json index 18551c6..91bd83f 100644 --- a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json @@ -83,9 +83,21 @@ "minimum": 0 }, "outcome": { - "type": "object" + "type": "object", + "properties": { + "http_status": {"type": "integer", "minimum": 100, "maximum": 599}, + "attempts": {"type": "array", "items": {"$ref": "#/$defs/http_attempt"}}, + "metadata": {"type": "object", "properties": { + "attempts": {"type": "array", "items": {"$ref": "#/$defs/http_attempt"}} + }} + } } }, + "$defs": { + "http_attempt": {"type": "object", "properties": { + "http_status": {"type": "integer", "minimum": 100, "maximum": 599} + }} + }, "required": [ "operation", "operation_key", diff --git a/src/viano_bug_analysis/telemetry/core.py b/src/viano_bug_analysis/telemetry/core.py index 751862f..44cb00a 100644 --- a/src/viano_bug_analysis/telemetry/core.py +++ b/src/viano_bug_analysis/telemetry/core.py @@ -367,10 +367,11 @@ class Telemetry: self._metric("vba_llm_failed_attempt_usage_unknown_total", len(attempts)) self.emit("llm_round", level="INFO" if status == "SUCCESS" else "WARNING", status=status, code=record.get("code"), extra={"phase": phase, "provider_key": identity(metadata.get("provider")), "model_key": identity(metadata.get("model")), + "http_status": record["outcome"].get("http_status"), "input_tokens": metadata.get("input_tokens"), "output_tokens": metadata.get("output_tokens"), "estimated_cost": cost if currency_valid else None, "currency": currency if currency_valid else None, - "failed_providers": [{"provider_key": identity(a.get("provider")), "code": safe_code(a.get("code"))} for a in attempts]}) + "failed_providers": [{"provider_key": identity(a.get("provider")), "code": safe_code(a.get("code")), "http_status": a.get("http_status")} for a in attempts]}) def refresh_inventory(self, *, force=False): if not self.started or not force and self.monotonic() - self._last_scan < self.config.inventory_interval_seconds: diff --git a/src/viano_bug_analysis/telemetry/safety.py b/src/viano_bug_analysis/telemetry/safety.py index 2a28141..d6fa6db 100644 --- a/src/viano_bug_analysis/telemetry/safety.py +++ b/src/viano_bug_analysis/telemetry/safety.py @@ -73,6 +73,10 @@ def number(value): return value if type(value) in {int, float} and math.isfinite(value) and value >= 0 else None +def http_status(value): + return value if type(value) is int and 100 <= value <= 599 else None + + def detail(event, value): """Strict field allowlists even for callers emitting their own events.""" if not isinstance(value, dict): @@ -93,11 +97,12 @@ def detail(event, value): "reason": safe_code(value.get("reason")) if value.get("reason") else None} if event == "llm_round": return {"phase": value.get("phase") if value.get("phase") in STATES else None, + "http_status": http_status(value.get("http_status")), "provider_key": hashed(value.get("provider_key")), "model_key": hashed(value.get("model_key")), "input_tokens": number(value.get("input_tokens")), "output_tokens": number(value.get("output_tokens")), "estimated_cost": number(value.get("estimated_cost")), "currency": value.get("currency") if isinstance(value.get("currency"), str) and re.fullmatch("[A-Z]{3}", value["currency"]) else None, - "failed_providers": [{"provider_key": hashed(a.get("provider_key")), "code": safe_code(a.get("code"))} for a in value.get("failed_providers", [])[:64] if isinstance(a, dict)]} + "failed_providers": [{"provider_key": hashed(a.get("provider_key")), "code": safe_code(a.get("code")), "http_status": http_status(a.get("http_status"))} for a in value.get("failed_providers", [])[:64] if isinstance(a, dict)]} if event in {"operation_failed", "phase_diagnostic"}: chains = [] for error in value.get("exceptions", [])[:8]: diff --git a/tests/test_llm_gateway.py b/tests/test_llm_gateway.py index a4e558f..8ac5091 100644 --- a/tests/test_llm_gateway.py +++ b/tests/test_llm_gateway.py @@ -181,7 +181,7 @@ def test_real_http_adapter(http_endpoint, monkeypatch, format): else: assert seen[0]["body"]["response_format"]["type"] == format -@pytest.mark.parametrize("status,temporary", [(401,False),(400,False),(429,True),(503,True),(302,False)]) +@pytest.mark.parametrize("status,temporary", [(401,False),(400,False),(408,True),(429,True),(500,True),(503,True),(302,False)]) def test_http_errors_and_redirects(http_endpoint,status,temporary): url, seen, reply = http_endpoint reply.update(status=status,body=b"secret error body",headers={"Location":url+"/redirect"}) @@ -190,6 +190,60 @@ def test_http_errors_and_redirects(http_endpoint,status,temporary): asyncio.run(LLMGateway(cfg).complete_round(request())) assert "secret" not in str(caught.value) assert len(seen) == 1 + if temporary: + assert caught.value.attempts[0]["http_status"] == status + else: + assert caught.value.http_status == status + + +@pytest.mark.parametrize("status", [401, 429, 503]) +def test_http_status_persisted_and_visible_in_safe_telemetry(make_package, tmp_path, status): + from viano_bug_analysis.telemetry import Telemetry + from viano_bug_analysis.telemetry.__main__ import read_events + from viano_bug_analysis.case.contracts import validate + workspace = Workspace(tmp_path / "work") + error = ProviderUnavailable("LLM_HTTP_TEMPORARY", http_status=status) if status != 401 else LLMError("LLM_HTTP_CONFIGURATION", http_status=status) + with Telemetry(workspace) as telemetry: + store = CheckpointStore(workspace, telemetry=telemetry) + store.intake(make_package()) + config = GatewayConfig((provider(),)) + gateway = LLMGateway(config, {"fixture": Adapter({"first": error})}) + result = asyncio.run(CaseLLMRunner(store, gateway).run("case-A", "stable-package", operation_id="http-status", request=request())) + with PackageReader(result.package) as reader: + record = json.loads(reader.read_bytes(result.result_ref)) + if status == 401: + assert record["outcome"]["http_status"] == status + del record["outcome"]["http_status"] + else: + assert record["outcome"]["attempts"][0]["http_status"] == status + del record["outcome"]["attempts"][0]["http_status"] + # Optional field: previously saved v1 records remain valid. + validate(record, "llm-call") + events = read_events(workspace.root) + event = next(e for e in events if e["event"] == "llm_round") + if status == 401: + assert event["detail"]["http_status"] == status + else: + assert event["detail"]["failed_providers"][0]["http_status"] == status + + +def test_successful_failover_keeps_first_http_status(): + config = GatewayConfig((provider("first"), provider("second"))) + adapter = Adapter({"first": ProviderUnavailable("LLM_HTTP_TEMPORARY", http_status=429), + "second": ProviderResponse({"answer": "ok"}, "fixture-model", 1, 1)}) + outcome = asyncio.run(LLMGateway(config, {"fixture": adapter}).complete_round(request())) + assert outcome["metadata"]["attempts"] == [ + {"provider": "first", "code": "LLM_HTTP_TEMPORARY", "http_status": 429}] + + +@pytest.mark.parametrize("value", [True, "secret-value", 99, 600, None]) +def test_http_status_only_accepts_bounded_integers(value): + from viano_bug_analysis.telemetry.safety import detail + assert LLMError("LLM_HTTP_CONFIGURATION", http_status=value).http_status is None + data = detail("llm_round", {"http_status": value, "failed_providers": [{"http_status": value}]}) + assert data["http_status"] is None + assert data["failed_providers"][0]["http_status"] is None + assert "secret-value" not in json.dumps(data) @pytest.mark.parametrize("body", [b'not JSON',b'{"choices":[],"choices":[]}',