Preserve bootstrap schema error paths for targeted correction
This commit is contained in:
1 parent
af2ee6cb91
commit
d957974498
7 files changed
+114
-5
No files matched your search
@@ -0,0 +1,77 @@
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from test_case_builder import case_setup
|
||||
from test_case_interpreter import gateway
|
||||
from test_investigation_tools import tool_case
|
||||
from test_parameter_and_bootstrap_guard import bootstrap
|
||||
from test_lean_investigation_decision import reply
|
||||
from viano_bug_analysis.llm import LLMError, ProviderResponse
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fault,path,keyword,fields', [
|
||||
('type','/update/working_brief/findings','type',[]),
|
||||
('required','/update/working_brief','required',['retired']),
|
||||
('extra','/update/working_brief','additionalProperties',['confidence']),
|
||||
])
|
||||
def test_bootstrap_gateway_reports_safe_nested_schema_location(fault,path,keyword,fields):
|
||||
document=reply();document['update']['next_step']=None
|
||||
brief=document['update']['working_brief']
|
||||
if fault=='type':brief['findings']='PRIVATE_RESPONSE_VALUE'
|
||||
elif fault=='required':brief.pop('retired')
|
||||
else:brief['confidence']='PRIVATE_RESPONSE_VALUE'
|
||||
class Adapter:
|
||||
async def generate(self,*args,**kwargs):
|
||||
return ProviderResponse(document,'fixture',100,50)
|
||||
with pytest.raises(LLMError) as caught:
|
||||
asyncio.run(gateway(Adapter()).complete_round(bootstrap()))
|
||||
assert caught.value.code=='LLM_OUTPUT_SCHEMA'
|
||||
details=caught.value.diagnostics
|
||||
assert details['validation_path']==path
|
||||
assert details['validation_keyword']==keyword
|
||||
assert details['validation_fields']==fields
|
||||
assert 'PRIVATE_RESPONSE_VALUE' not in json.dumps(details)
|
||||
assert caught.value.metadata['input_tokens']==100
|
||||
|
||||
|
||||
def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool_case,tmp_path):
|
||||
from test_investigation_hybrid import HybridAdapter
|
||||
from test_working_brief import service_for
|
||||
from test_investigator import CASE,state
|
||||
from viano_bug_analysis.analysis import Investigator,InvestigationConfig
|
||||
from viano_bug_analysis.budget import BudgetManager
|
||||
from viano_bug_analysis.lifecycle import CheckpointStore,Workspace
|
||||
from viano_bug_analysis.package import PackageReader,PackageValidator
|
||||
from viano_bug_analysis.tools import InvestigationToolRouter
|
||||
class Adapter(HybridAdapter):
|
||||
def __init__(self):super().__init__();self.sent=[]
|
||||
async def generate(self,config,request,**kwargs):
|
||||
self.sent.append(copy.deepcopy(request))
|
||||
assert request['prompt_version']=='investigation_bootstrap_v21'
|
||||
response=await super().generate(config,dict(request,prompt_version='investigation_project_v18'),**kwargs)
|
||||
if len(self.sent)==1:
|
||||
response.document['update']['working_brief']['findings']='PRIVATE_RESPONSE_VALUE'
|
||||
return response
|
||||
adapter=Adapter();router=InvestigationToolRouter(project_research=service_for(tool_case))
|
||||
loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig(
|
||||
lean_loop=True,lean_protocol=2,lean_checkpoint_steps=0,bootstrap_json_framing=True),router=router)
|
||||
first=asyncio.run(loop.step(*CASE))
|
||||
assert first.status=='REJECTED' and first.code=='LLM_OUTPUT_SCHEMA'
|
||||
assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions']
|
||||
with PackageReader(first.package) as reader:
|
||||
context=json.loads(reader.read_bytes(state(first)['active']['context_ref']))
|
||||
call=json.loads(reader.read_bytes(state(first)['hypotheses']['H1']['notes'][-1]['llm_ref']))
|
||||
feedback=context['previous_rejection_details']
|
||||
assert feedback=={'validation_path':'/update/working_brief/findings','validation_fields':[],
|
||||
'requirement':'match_expected_type_at_validation_path'}
|
||||
assert call['outcome']['diagnostics']['validation_path']==feedback['validation_path']
|
||||
assert 'PRIVATE_RESPONSE_VALUE' not in json.dumps(call)
|
||||
portable=CheckpointStore(Workspace(tmp_path/'portable'));portable.intake(first.package)
|
||||
result=asyncio.run(Investigator(portable,gateway(adapter),router=router).step(*CASE))
|
||||
assert result.status=='PROGRESSED' and len(adapter.sent)==2
|
||||
assert 'match_expected_type_at_validation_path' in json.dumps(adapter.sent[1])
|
||||
assert '/update/working_brief/findings' in json.dumps(adapter.sent[1])
|
||||
assert PackageValidator().validate(result.package).valid
|
||||
Reference in new issue
Block a user