Derive final diagnosis references and provide targeted correction feedback
This commit is contained in:
1 parent
49095ac803
commit
cf69aa3aa9
10 files changed
+513
-28
No files matched your search
@@ -70,13 +70,13 @@ def test_new_diagnosis_keeps_frozen_context_and_uses_compact_request(investigate
|
||||
adapter = FinalAdapter(True)
|
||||
result = build(DiagnosisBuilder(store, gateway(adapter)))
|
||||
request = adapter.requests[0]
|
||||
assert request['prompt_version'] == 'diagnosis_v5'
|
||||
assert request['prompt_version'] == 'diagnosis_v6'
|
||||
visible = restore(json.loads(request['messages'][1]['content'])['input'])
|
||||
with PackageReader(result.package) as reader:
|
||||
doc = json.loads(reader.read_bytes(result.diagnosis_ref))
|
||||
context = json.loads(reader.read_bytes(doc['context_ref']))
|
||||
assert 'inputs' in context['case'] and 'inputs' not in visible['case']
|
||||
assert request['output_schema'] == build_prompt(context, modern=True)['output_schema']
|
||||
assert request['output_schema'] == build_prompt(context, modern=True, lean=True)['output_schema']
|
||||
|
||||
|
||||
def test_handoff_v4_accepts_reviewed_remediation_data_without_rewriting_old_prompt():
|
||||
|
||||
@@ -218,9 +218,58 @@ def test_legacy_rejection_upgrades_next_attempt_without_rewriting_context(invest
|
||||
assert build(builder).status=='REJECTED'
|
||||
assert adapter.requests[0]['prompt_version']=='diagnosis_v1'
|
||||
assert build(builder).status=='COMPLETED'
|
||||
assert adapter.requests[1]['prompt_version']=='diagnosis_v5'
|
||||
assert adapter.requests[1]['prompt_version']=='diagnosis_v6'
|
||||
received=restore(json.loads(adapter.requests[1]['messages'][1]['content'])['input'])
|
||||
assert received['validation_feedback'][0]['code']=='DIAGNOSIS_SELECTION'
|
||||
assert build(builder).reused and adapter.calls==2
|
||||
with PackageReader(store.latest(*CASE)) as reader:assert reader.read_bytes(ref)==raw
|
||||
assert budget.snapshot(*CASE).state==snapshot.state
|
||||
|
||||
|
||||
def test_lean_reply_derivation_and_portable_replay(investigated,tmp_path):
|
||||
store,_ = investigated
|
||||
def change(payload,context):
|
||||
payload.pop('reason')
|
||||
payload['evaluations'][0]['evidence_refs'] = []
|
||||
adapter = FinalAdapter(True,change)
|
||||
builder = DiagnosisBuilder(store,gateway(adapter))
|
||||
result = build(builder)
|
||||
assert result.status == 'COMPLETED'
|
||||
assert adapter.requests[0]['prompt_version'] == 'diagnosis_v6'
|
||||
with PackageReader(result.package) as reader:
|
||||
doc=json.loads(reader.read_bytes(result.diagnosis_ref))
|
||||
call=json.loads(reader.read_bytes(doc['llm_result_ref']))
|
||||
assert 'reason' not in call['outcome']['document']
|
||||
assert call['outcome']['document']['evaluations'][0]['evidence_refs'] == []
|
||||
assert doc['diagnosis']['evaluations'][0]['evidence_refs'] == doc['diagnosis']['root_cause']['evidence_refs']
|
||||
portable=CheckpointStore(Workspace(tmp_path/'lean-portable'));portable.intake(result.package)
|
||||
assert build(DiagnosisBuilder(portable,gateway(adapter))).reused
|
||||
assert adapter.calls == 1
|
||||
|
||||
|
||||
def test_targeted_feedback_after_unknown_evidence(investigated):
|
||||
store,_=investigated
|
||||
def change(payload,context):
|
||||
payload['root_cause']['evidence_refs']=['E999999']
|
||||
adapter=FinalAdapter(True,change);builder=DiagnosisBuilder(store,gateway(adapter))
|
||||
assert build(builder).status=='REJECTED'
|
||||
adapter.change=None
|
||||
assert build(builder).status=='COMPLETED'
|
||||
visible=restore(json.loads(adapter.requests[-1]['messages'][1]['content'])['input'])
|
||||
assert visible['validation_feedback'][-1]['details']['requirement']=='cite_only_available_evidence_ids'
|
||||
|
||||
|
||||
def test_frozen_v5_request_replays_without_lean_derivation(investigated):
|
||||
store,_=investigated
|
||||
adapter=FinalAdapter(True)
|
||||
builder=DiagnosisBuilder(store,gateway(adapter))
|
||||
original=builder._request
|
||||
def frozen(context,reader,attempt,**kwargs):
|
||||
kwargs.update(compact=True,consolidated=True,selected=True,lean=False)
|
||||
return original(context,reader,attempt,**kwargs)
|
||||
builder._request=frozen
|
||||
result=build(builder)
|
||||
assert result.status=='COMPLETED'
|
||||
assert adapter.requests[0]['prompt_version']=='diagnosis_v5'
|
||||
assert build(DiagnosisBuilder(store,gateway(adapter))).reused
|
||||
assert adapter.calls==1
|
||||
@@ -91,3 +91,29 @@ def test_supported_alternative_is_not_automatically_a_contradiction():
|
||||
saved, ctx, decision, budget = gate_fixture()
|
||||
saved['hypotheses']['H2']['confidence'] = 'supported'
|
||||
check_request(saved, ctx, decision, budget)
|
||||
|
||||
|
||||
def test_lean_normalization_adds_root_refs_without_mutating_raw_reply():
|
||||
from viano_bug_analysis.analysis.diagnosis_builder import normalize_evaluation
|
||||
payload, context = example()
|
||||
context['evidence']['E000004'] = {'content': {'items':[{'column':'guid'}], 'coverage':'schema_only'}}
|
||||
payload['root_cause']['evidence_refs'].append('E000004')
|
||||
payload.pop('reason')
|
||||
original = copy.deepcopy(payload)
|
||||
result = normalize_evaluation(payload)
|
||||
validate_evaluation(result, context)
|
||||
assert result['evaluations'][0]['evidence_refs'] == ['E000001','E000004']
|
||||
assert result['reason'] == payload['decision_basis']['causal_explanation']
|
||||
assert payload == original
|
||||
assert normalize_evaluation(result) == result
|
||||
|
||||
|
||||
@pytest.mark.parametrize('mode', ['unknown','schema_only','contradiction'])
|
||||
def test_normalization_does_not_bypass_material_controls(mode):
|
||||
from viano_bug_analysis.analysis.diagnosis_builder import normalize_evaluation
|
||||
payload, context = example()
|
||||
if mode == 'unknown': payload['root_cause']['evidence_refs'] = ['E999999']
|
||||
if mode == 'schema_only': context['evidence']['E000001']['content']['coverage'] = 'schema_only'
|
||||
if mode == 'contradiction':
|
||||
payload['decision_basis']['contradictions'] = [{'evidence_refs':['E000001'], 'resolution':'unresolved','reason':'Runtime contradicts this cause.'}]
|
||||
with pytest.raises(DiagnosisError): validate_evaluation(normalize_evaluation(payload), context)
|
||||
+1
-1
@@ -28,7 +28,7 @@ class PipelineAdapter:
|
||||
async def generate(self, config, request, **kwargs):
|
||||
self.calls += 1
|
||||
phase = {"case_interpretation_v1": "case", "hypotheses_v1": "hypotheses", "hypotheses_v2": "hypotheses", "hypotheses_v3": "hypotheses", "hypotheses_v4": "hypotheses",
|
||||
"investigation_v1": "loop", "investigation_v2": "loop", "investigation_v3": "loop", "diagnosis_v1": "diagnosis", "diagnosis_v2": "diagnosis", "diagnosis_v3": "diagnosis", "diagnosis_v4": "diagnosis", "diagnosis_v5": "diagnosis",
|
||||
"investigation_v1": "loop", "investigation_v2": "loop", "investigation_v3": "loop", "diagnosis_v1": "diagnosis", "diagnosis_v2": "diagnosis", "diagnosis_v3": "diagnosis", "diagnosis_v4": "diagnosis", "diagnosis_v5": "diagnosis", "diagnosis_v6": "diagnosis",
|
||||
"change_request_v1": "request", "change_request_v2": "request", "change_request_v3": "request", "change_request_v4": "request", "change_request_v5": "request", "change_request_v6": "request"}[request["prompt_version"]]
|
||||
adapter = self.adapters[phase]
|
||||
return await adapter.generate(config, request, **kwargs)
|
||||
|
||||
Reference in new issue
Block a user