Add configurable LLM gateway with ordered failover and durable retries

src/viano_bug_analysis/llm/gateway.py

@@ -0,0 +1,106 @@
+import asyncio
+import copy
+import logging
+import time
+
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import SchemaError, ValidationError
+
+from viano_bug_analysis.lifecycle.storage import json_bytes
+from viano_bug_analysis.package.manifest import FORMAT_CHECKER, parse_manifest
+
+from .errors import GatewayUnavailable, LLMError, ProviderUnavailable
+from .openai_compatible import OpenAICompatibleProvider
+
+LOGGER = logging.getLogger("viano_bug_analysis.llm")
+PRIVACY = {"local_only": {"local"}, "self_hosted": {"local", "self_hosted"}, "cloud_allowed": {"local", "self_hosted", "cloud"}}
+
+
+def _schema_check(schema):
+    # No external resolution/network access during validation, including nested refs.
+    if isinstance(schema, dict):
+        for key, value in schema.items():
+            if key in {"$ref", "$dynamicRef"} and (not isinstance(value, str) or not value.startswith("#")):
+                raise LLMError("LLM_SCHEMA_REFERENCE")
+            _schema_check(value)
+    elif isinstance(schema, list):
+        for value in schema:
+            _schema_check(value)
+
+
+class LLMGateway:
+    def __init__(self, config, adapters=None):
+        self.config = config
+        self.adapters = {"openai_compatible": OpenAICompatibleProvider()} | (adapters or {})
+        if any(p.enabled and p.adapter not in self.adapters for p in config.providers):
+            raise LLMError("LLM_ADAPTER")
+
+    def validate_request(self, request):
+        try:
+            encoded = json_bytes(request)
+            if len(encoded) > self.config.max_request_bytes:
+                raise LLMError("LLM_REQUEST_LIMIT")
+            request = parse_manifest(encoded)
+            if not isinstance(request, dict) or set(request) != {"prompt_version", "privacy", "messages", "output_schema"}:
+                raise LLMError("LLM_REQUEST")
+            if request["privacy"] == "configured":
+                request["privacy"] = self.config.privacy_profile
+            if request["privacy"] not in PRIVACY or not isinstance(request["prompt_version"], str) or not request["prompt_version"].strip():
+                raise LLMError("LLM_REQUEST")
+            if not isinstance(request["messages"], list) or not request["messages"]:
+                raise LLMError("LLM_REQUEST")
+            for message in request["messages"]:
+                if not isinstance(message, dict) or set(message) != {"role", "content"} or message["role"] not in {"system", "user", "assistant"} or not isinstance(message["content"], str):
+                    raise LLMError("LLM_REQUEST")
+            if not isinstance(request["output_schema"], dict) or request["output_schema"].get("type") != "object":
+                raise LLMError("LLM_REQUEST")
+            _schema_check(request["output_schema"])
+            Draft202012Validator.check_schema(request["output_schema"])
+            eligible = PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]
+            if not any(p.enabled and p.privacy in eligible for p in self.config.providers):
+                raise LLMError("LLM_PRIVACY_NO_PROVIDER")
+            return request
+        except (ValueError, TypeError, RecursionError, SchemaError):
+            raise LLMError("LLM_REQUEST") from None
+
+    async def complete_round(self, request: dict) -> dict:
+        """Try eligible providers in order exactly once; never sleep or retry internally."""
+        request = self.validate_request(request)
+        attempts = []
+        for provider in self.config.providers:
+            if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]):
+                continue
+            started = time.monotonic()
+            try:
+                response = await asyncio.wait_for(self.adapters[provider.adapter].generate(provider, copy.deepcopy(request), max_response_bytes=self.config.max_response_bytes), timeout=provider.timeout_seconds)
+                if not isinstance(response.model, str) or not response.model.strip() or len(response.model) > 256 or any(ord(c) < 32 for c in response.model):
+                    raise LLMError("LLM_RESPONSE")
+                for count in (response.input_tokens, response.output_tokens):
+                    if count is not None and (isinstance(count, bool) or not isinstance(count, int) or not 0 <= count <= 10**12):
+                        raise LLMError("LLM_RESPONSE")
+                encoded = json_bytes(response.document)
+                if len(encoded) > self.config.max_response_bytes:
+                    raise LLMError("LLM_RESPONSE_LIMIT")
+                if not isinstance(response.document, dict):
+                    raise LLMError("LLM_OUTPUT_SCHEMA")
+                Draft202012Validator(request["output_schema"], format_checker=FORMAT_CHECKER).validate(response.document)
+                cost = None
+                if all(value is not None for value in (provider.input_cost_per_million, provider.output_cost_per_million, response.input_tokens, response.output_tokens)):
+                    cost = (response.input_tokens * provider.input_cost_per_million + response.output_tokens * provider.output_cost_per_million) / 1_000_000
+                metadata = {"provider": provider.id, "model": response.model, "prompt_version": request["prompt_version"],
+                            "input_tokens": response.input_tokens, "output_tokens": response.output_tokens,
+                            "estimated_cost": cost, "elapsed_seconds": round(time.monotonic() - started, 6), "attempts": attempts}
+                metadata["cost_currency"] = provider.cost_currency if cost is not None else None
+                LOGGER.info("LLM request succeeded provider=%s", provider.id)
+                return {"document": parse_manifest(encoded), "metadata": metadata}
+            except (ProviderUnavailable, TimeoutError) as error:
+                code = error.code if isinstance(error, ProviderUnavailable) else "LLM_TIMEOUT"
+                attempts.append({"provider": provider.id, "code": code})
+                LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code)
+            except ValidationError:
+                raise LLMError("LLM_OUTPUT_SCHEMA") from None
+            except LLMError:
+                raise
+            except Exception:
+                raise LLMError("LLM_PROVIDER_ERROR") from None
+        raise GatewayUnavailable(attempts)