Add traceable case interpretation with validated LLM output

This commit is contained in:
Nico committed 2026-10-13 22:34:23 +02:00
1 parent b2b77c7723
commit b93e1acb22
18 files changed
+1826 -7

No files matched your search

+23
View File
@@ -10,6 +10,9 @@ from viano_bug_analysis.case import CaseBuilder
from viano_bug_analysis.lifecycle import CheckpointStore, Workspace
from viano_bug_analysis.package import PackageReader, PackageValidator
from viano_bug_analysis.profiles.vcore import ArchiveReader, VcoreImporter
from viano_bug_analysis.analysis import CaseInterpreter
from test_case_interpreter import FixtureAdapter, gateway, missing_payload
import asyncio
@pytest.mark.skipif(not os.environ.get("VBA_WOWDIAG_SAMPLE") or not os.environ.get("VBA_WOWSRC_SAMPLE"),
@@ -37,4 +40,24 @@ def test_full_bug20_package_pair(tmp_path):
assert len(matching) == 1
assert reader.read_bytes(matching[0]) == source.read_member(member)
assert CaseBuilder(store).build("regression-case", "regression-package").reused
# Offline phase integration only: this fixture is not a test of model reasoning.
response = missing_payload()
response["observed_behavior"][0]["text"] = case["human_observations"]["observations"][0]["text"]
adapter = FixtureAdapter(response)
interpreter = CaseInterpreter(store, gateway(adapter))
interpreted = asyncio.run(interpreter.interpret("regression-case", "regression-package"))
assert interpreted.status == "COMPLETED"
with PackageReader(result.package) as before, PackageReader(interpreted.package) as after:
document = json.loads(after.read_bytes(interpreted.interpretation_ref))
assert document["expected_behavior_completeness"] == "missing"
assert document["interpretation"]["expected_behavior"] == []
assert document["interpretation"]["behavior_gap"] is None
assert after.manifest["artifacts"]["diagnosis"] is None
assert after.manifest["artifacts"]["change_request"] is None
for name in before.files:
if name != "manifest.json":
assert before.read_bytes(name) == after.read_bytes(name)
assert PackageValidator().validate(interpreted.package).valid
assert asyncio.run(interpreter.interpret("regression-case", "regression-package")).reused
assert len(adapter.requests) == 1
assert [hashlib.sha256(path.read_bytes()).hexdigest() for path in (diag, src)] == original_hashes