89 lines
5.1 KiB
Python
89 lines
5.1 KiB
Python
import asyncio
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from jsonschema import Draft202012Validator
|
|
from test_case_interpreter import built_case
|
|
from test_case_builder import case_setup
|
|
from test_entity_resolution import Model,Database,PLAN,service_for,run,gateway
|
|
from viano_bug_analysis.analysis.entity_resolution import (
|
|
assessment_feedback,project_assessment,validate_assessment,request_for,policy)
|
|
from viano_bug_analysis.analysis.errors import InterpretationError
|
|
from viano_bug_analysis.llm import ProviderResponse
|
|
from viano_bug_analysis.lifecycle import CheckpointStore
|
|
from viano_bug_analysis.package import PackageReader
|
|
|
|
|
|
def corpus():
|
|
return json.loads(Path('tests/fixtures/llm_rejections/20261024/bug21-localized-no-match/validation-case.json').read_text(encoding='utf-8'))
|
|
|
|
|
|
def test_real_rejection_yields_scoped_repair_and_retains_furien():
|
|
d=corpus();before=copy.deepcopy(d)
|
|
feedback=assessment_feedback(d['answer'],d['plan'],d['captures'],d['candidates'],d['context'])
|
|
assert feedback['locked_resolutions']==[d['answer']['resolutions'][0]]
|
|
assert len(feedback['issues'])==1
|
|
issue=feedback['issues'][0]
|
|
assert issue['search_index']==1 and issue['returned_rows']==8 and issue['truncated']
|
|
assert 'zero returned rows' in issue['instruction']
|
|
replacement=copy.deepcopy(d['answer']['resolutions'][1]);replacement['status']='ambiguous'
|
|
result=project_assessment({'resolutions':[replacement]},feedback)
|
|
validate_assessment(result,d['plan'],d['captures'],d['candidates'],d['context'])
|
|
assert result['resolutions'][0]==d['answer']['resolutions'][0]
|
|
with pytest.raises(InterpretationError):project_assessment(d['answer'],feedback)
|
|
assert d==before
|
|
|
|
|
|
@pytest.mark.parametrize('bad', ['no_match','selected','source'])
|
|
def test_invalid_repaired_entry_is_still_rejected(bad):
|
|
d=corpus();feedback=assessment_feedback(d['answer'],d['plan'],d['captures'],d['candidates'],d['context'])
|
|
replacement=copy.deepcopy(d['answer']['resolutions'][1])
|
|
if bad=='selected':replacement.update(status='selected',candidate_id='K9999')
|
|
if bad=='source':replacement.update(status='ambiguous',source_ids=['F9999'])
|
|
combined=project_assessment({'resolutions':[replacement]},feedback)
|
|
with pytest.raises(InterpretationError):validate_assessment(combined,d['plan'],d['captures'],d['candidates'],d['context'])
|
|
|
|
|
|
@pytest.mark.parametrize('malformed_patch',[False,True])
|
|
def test_two_search_repair_survives_restart_and_portable_replay(built_case,tmp_path,malformed_patch):
|
|
plan=copy.deepcopy(PLAN)
|
|
plan['searches'].append({**plan['searches'][0],'catalog':'localized'})
|
|
class PartialModel(Model):
|
|
def __init__(self):super().__init__(plan=plan);self.repairs=0
|
|
async def generate(self,config,request,**kwargs):
|
|
if request['prompt_version']=='entity_resolution_assessment_v4':
|
|
wire=json.loads(request['messages'][1]['content'])
|
|
if 'validation_feedback' in wire:
|
|
self.requests.append(copy.deepcopy(request));self.repairs+=1
|
|
feedback=wire['validation_feedback']['citation_corrections']
|
|
assert feedback['locked_resolutions'][0]['candidate_id']=='K0001'
|
|
assert [i['search_index'] for i in feedback['issues']]==[1]
|
|
out={'resolutions':[{'search_index':1,'status':'ambiguous','candidate_id':None,
|
|
'reason':'Unresolved localized identity','source_ids':['O0001','V0002']}]}
|
|
assert Draft202012Validator(request['output_schema']).is_valid(out)
|
|
assert 'REPAIR SCOPE OVERRIDE' in request['messages'][0]['content']
|
|
if malformed_patch and self.repairs==1:out['resolutions'][0]['search_index']=0
|
|
return ProviderResponse(out,'fixture',100,50)
|
|
response=await super().generate(config,request,**kwargs)
|
|
response.document['resolutions'][1].update(status='no_match',candidate_id=None)
|
|
return response
|
|
return await super().generate(config,request,**kwargs)
|
|
store=built_case[0];service=service_for(store,Database());model=PartialModel()
|
|
service.entity_resolution['catalogs'].append({**service.entity_resolution['catalogs'][0],'id':'localized'})
|
|
assert run(store,model,service).status=='REJECTED'
|
|
if malformed_patch:assert run(store,model,service).status=='REJECTED'
|
|
result=run(store,model,service)
|
|
# Identical fixture queries share the existing research cache.
|
|
assert result.status=='COMPLETED' and len(service.database.calls)==1
|
|
with PackageReader(result.package) as reader:
|
|
assessment=json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json'))['payload']['document']
|
|
assert assessment['resolutions'][0]['candidate_id']=='K0001'
|
|
assert assessment['resolutions'][1]['status']=='ambiguous'
|
|
recipient=CheckpointStore(type(store.workspace)(tmp_path/'recipient'))
|
|
recipient.intake(result.package)
|
|
count=len(model.requests)
|
|
assert run(recipient,model).reused
|
|
assert len(model.requests)==count
|