Preserve HTTP status codes in LLM history and safe telemetry
This commit is contained in:
1 parent
5826d8f730
commit
4d0edc030d
10 files changed
+99
-7
No files matched your search
@@ -181,7 +181,7 @@ def test_real_http_adapter(http_endpoint, monkeypatch, format):
|
||||
else: assert seen[0]["body"]["response_format"]["type"] == format
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,temporary", [(401,False),(400,False),(429,True),(503,True),(302,False)])
|
||||
@pytest.mark.parametrize("status,temporary", [(401,False),(400,False),(408,True),(429,True),(500,True),(503,True),(302,False)])
|
||||
def test_http_errors_and_redirects(http_endpoint,status,temporary):
|
||||
url, seen, reply = http_endpoint
|
||||
reply.update(status=status,body=b"secret error body",headers={"Location":url+"/redirect"})
|
||||
@@ -190,6 +190,60 @@ def test_http_errors_and_redirects(http_endpoint,status,temporary):
|
||||
asyncio.run(LLMGateway(cfg).complete_round(request()))
|
||||
assert "secret" not in str(caught.value)
|
||||
assert len(seen) == 1
|
||||
if temporary:
|
||||
assert caught.value.attempts[0]["http_status"] == status
|
||||
else:
|
||||
assert caught.value.http_status == status
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [401, 429, 503])
|
||||
def test_http_status_persisted_and_visible_in_safe_telemetry(make_package, tmp_path, status):
|
||||
from viano_bug_analysis.telemetry import Telemetry
|
||||
from viano_bug_analysis.telemetry.__main__ import read_events
|
||||
from viano_bug_analysis.case.contracts import validate
|
||||
workspace = Workspace(tmp_path / "work")
|
||||
error = ProviderUnavailable("LLM_HTTP_TEMPORARY", http_status=status) if status != 401 else LLMError("LLM_HTTP_CONFIGURATION", http_status=status)
|
||||
with Telemetry(workspace) as telemetry:
|
||||
store = CheckpointStore(workspace, telemetry=telemetry)
|
||||
store.intake(make_package())
|
||||
config = GatewayConfig((provider(),))
|
||||
gateway = LLMGateway(config, {"fixture": Adapter({"first": error})})
|
||||
result = asyncio.run(CaseLLMRunner(store, gateway).run("case-A", "stable-package", operation_id="http-status", request=request()))
|
||||
with PackageReader(result.package) as reader:
|
||||
record = json.loads(reader.read_bytes(result.result_ref))
|
||||
if status == 401:
|
||||
assert record["outcome"]["http_status"] == status
|
||||
del record["outcome"]["http_status"]
|
||||
else:
|
||||
assert record["outcome"]["attempts"][0]["http_status"] == status
|
||||
del record["outcome"]["attempts"][0]["http_status"]
|
||||
# Optional field: previously saved v1 records remain valid.
|
||||
validate(record, "llm-call")
|
||||
events = read_events(workspace.root)
|
||||
event = next(e for e in events if e["event"] == "llm_round")
|
||||
if status == 401:
|
||||
assert event["detail"]["http_status"] == status
|
||||
else:
|
||||
assert event["detail"]["failed_providers"][0]["http_status"] == status
|
||||
|
||||
|
||||
def test_successful_failover_keeps_first_http_status():
|
||||
config = GatewayConfig((provider("first"), provider("second")))
|
||||
adapter = Adapter({"first": ProviderUnavailable("LLM_HTTP_TEMPORARY", http_status=429),
|
||||
"second": ProviderResponse({"answer": "ok"}, "fixture-model", 1, 1)})
|
||||
outcome = asyncio.run(LLMGateway(config, {"fixture": adapter}).complete_round(request()))
|
||||
assert outcome["metadata"]["attempts"] == [
|
||||
{"provider": "first", "code": "LLM_HTTP_TEMPORARY", "http_status": 429}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [True, "secret-value", 99, 600, None])
|
||||
def test_http_status_only_accepts_bounded_integers(value):
|
||||
from viano_bug_analysis.telemetry.safety import detail
|
||||
assert LLMError("LLM_HTTP_CONFIGURATION", http_status=value).http_status is None
|
||||
data = detail("llm_round", {"http_status": value, "failed_providers": [{"http_status": value}]})
|
||||
assert data["http_status"] is None
|
||||
assert data["failed_providers"][0]["http_status"] is None
|
||||
assert "secret-value" not in json.dumps(data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [b'not JSON',b'{"choices":[],"choices":[]}',
|
||||
|
||||
Reference in new issue
Block a user