Add traceable case interpretation with validated LLM output
This commit is contained in:
1 parent
b2b77c7723
commit
b93e1acb22
18 files changed
+1826
-7
No files matched your search
@@ -10,6 +10,9 @@ from viano_bug_analysis.case import CaseBuilder
|
||||
from viano_bug_analysis.lifecycle import CheckpointStore, Workspace
|
||||
from viano_bug_analysis.package import PackageReader, PackageValidator
|
||||
from viano_bug_analysis.profiles.vcore import ArchiveReader, VcoreImporter
|
||||
from viano_bug_analysis.analysis import CaseInterpreter
|
||||
from test_case_interpreter import FixtureAdapter, gateway, missing_payload
|
||||
import asyncio
|
||||
|
||||
|
||||
@pytest.mark.skipif(not os.environ.get("VBA_WOWDIAG_SAMPLE") or not os.environ.get("VBA_WOWSRC_SAMPLE"),
|
||||
@@ -37,4 +40,24 @@ def test_full_bug20_package_pair(tmp_path):
|
||||
assert len(matching) == 1
|
||||
assert reader.read_bytes(matching[0]) == source.read_member(member)
|
||||
assert CaseBuilder(store).build("regression-case", "regression-package").reused
|
||||
# Offline phase integration only: this fixture is not a test of model reasoning.
|
||||
response = missing_payload()
|
||||
response["observed_behavior"][0]["text"] = case["human_observations"]["observations"][0]["text"]
|
||||
adapter = FixtureAdapter(response)
|
||||
interpreter = CaseInterpreter(store, gateway(adapter))
|
||||
interpreted = asyncio.run(interpreter.interpret("regression-case", "regression-package"))
|
||||
assert interpreted.status == "COMPLETED"
|
||||
with PackageReader(result.package) as before, PackageReader(interpreted.package) as after:
|
||||
document = json.loads(after.read_bytes(interpreted.interpretation_ref))
|
||||
assert document["expected_behavior_completeness"] == "missing"
|
||||
assert document["interpretation"]["expected_behavior"] == []
|
||||
assert document["interpretation"]["behavior_gap"] is None
|
||||
assert after.manifest["artifacts"]["diagnosis"] is None
|
||||
assert after.manifest["artifacts"]["change_request"] is None
|
||||
for name in before.files:
|
||||
if name != "manifest.json":
|
||||
assert before.read_bytes(name) == after.read_bytes(name)
|
||||
assert PackageValidator().validate(interpreted.package).valid
|
||||
assert asyncio.run(interpreter.interpret("regression-case", "regression-package")).reused
|
||||
assert len(adapter.requests) == 1
|
||||
assert [hashlib.sha256(path.read_bytes()).hexdigest() for path in (diag, src)] == original_hashes
|
||||
Reference in new issue
Block a user